/// <summary>
/// Parses the benchmark statistics from the plain text line.
///
/// E.g. given the input <paramref name="line"/>:
///
/// Target 1: 10 op, 1005.8 ms, 1005842518 ns, 3332139 ticks, 100584251.7955 ns/op, 9.9 op/s
///
/// Will extract the number of <see cref="Operations"/> performed and the
/// total number of <see cref="Nanoseconds"/> it took to perform them.
/// </summary>
/// <param name="logger">The logger to write any diagnostic messages to.</param>
/// <param name="line">The line to parse.</param>
/// <returns>An instance of <see cref="BenchmarkRunReport"/> if parsed successfully. <c>Null</c> in case of any trouble.</returns>
public static BenchmarkRunReport Parse(IBenchmarkLogger logger, string line)
{
try
{
var op = 1L;
var ns = double.PositiveInfinity;
var items = line.
Split(new[] {':'}, StringSplitOptions.RemoveEmptyEntries)[1].
Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries);
foreach (var item in items)
{
var split = item.Split(new[] {' '}, StringSplitOptions.RemoveEmptyEntries);
var unit = split[1];
switch (unit)
{
case "ns":
ns = double.Parse(split[0], EnvironmentHelper.MainCultureInfo);
break;
case "op":
op = long.Parse(split[0]);
break;
}
}
return new BenchmarkRunReport(op, ns);
}
catch (Exception)
{
logger.WriteLineError("Parse error in the following line:");
logger.WriteLineError(line);
return null;
}
}
/// <summary>
/// Parses the benchmark statistics from the plain text line.
///
/// E.g. given the input <paramref name="line"/>:
///
/// Target 1: 10 op, 1005.8 ms, 1005842518 ns, 3332139 ticks, 100584251.7955 ns/op, 9.9 op/s
///
/// Will extract the number of <see cref="Operations"/> performed and the
/// total number of <see cref="Nanoseconds"/> it took to perform them.
/// </summary>
/// <param name="logger">The logger to write any diagnostic messages to.</param>
/// <param name="line">The line to parse.</param>
/// <returns>An instance of <see cref="BenchmarkRunReport"/> if parsed successfully. <c>Null</c> in case of any trouble.</returns>
public static BenchmarkRunReport Parse(IBenchmarkLogger logger, string line)
{
try
{
var op = 1L;
var ns = double.PositiveInfinity;
var items = line.
Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries)[1].
Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var item in items)
{
var split = item.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
var unit = split[1];
switch (unit)
{
case "ns":
ns = double.Parse(split[0], EnvironmentHelper.MainCultureInfo);
break;
case "op":
op = long.Parse(split[0]);
break;
}
}
return(new BenchmarkRunReport(op, ns));
}
catch (Exception)
{
logger.WriteLineError("Parse error in the following line:");
logger.WriteLineError(line);
return(null);
}
}
private void PrintTable(List<string[]> table, IBenchmarkLogger logger, string[] columnsToAlwaysShow)
{
if (table.Count == 0)
{
logger.WriteLineError("There are no found benchmarks");
logger.NewLine();
return;
}
int rowCount = table.Count, colCount = table[0].Length;
var columnsToShowIndexes = columnsToAlwaysShow.Select(col => Array.IndexOf(table[0], col)).ToArray();
int[] widths = new int[colCount];
bool[] areSame = new bool[colCount];
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
areSame[colIndex] = rowCount > 2 && colIndex < colCount;
for (int rowIndex = 0; rowIndex < rowCount; rowIndex++)
{
widths[colIndex] = Math.Max(widths[colIndex], table[rowIndex][colIndex].Length + 1);
if (rowIndex > 1 && table[rowIndex][colIndex] != table[1][colIndex])
areSame[colIndex] = false;
}
}
if (areSame.Any(s => s))
{
var paramsOnLine = 0;
for (int colIndex = 0; colIndex < colCount; colIndex++)
if (areSame[colIndex] && columnsToShowIndexes.Contains(colIndex) == false)
{
logger.WriteInfo($"{table[0][colIndex]}={table[1][colIndex]} ");
paramsOnLine++;
if (paramsOnLine == 3)
{
logger.NewLine();
paramsOnLine = 0;
}
}
logger.NewLine();
}
if (useCodeBlocks)
{
logger.Write("```");
logger.NewLine();
}
table.Insert(1, widths.Select(w => new string('-', w)).ToArray());
foreach (var row in table)
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
if (!areSame[colIndex] || columnsToShowIndexes.Contains(colIndex))
logger.WriteStatistic(row[colIndex].PadLeft(widths[colIndex], ' ') + " |");
logger.NewLine();
}
if (areSame.Any(s => s))
{
logger.NewLine();
}
}
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger)
{
logger.WriteLineInfo("```ini");
logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
var table = ReportExporterHelper.BuildTable(reports);
PrintTable(table, logger);
var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
}
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger)
{
logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
var table = BenchmarkExporterHelper.BuildTable(reports);
// If we have Benchmarks with ParametersSets, force the "Method" columns to be displayed, otherwise it doesn't make as much sense
var columnsToAlwaysShow = reports.Any(r => r.Benchmark.Task.ParametersSets != null) ? new[] { "Method" } : new string[0];
PrintTable(table, logger, columnsToAlwaysShow);
var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
}
private void PrintTable(List <string[]> table, IBenchmarkLogger logger, string[] columnsToAlwaysShow)
{
if (table.Count == 0)
{
logger.WriteLineError("There are no found benchmarks");
logger.NewLine();
return;
}
int rowCount = table.Count, colCount = table[0].Length;
var columnsToShowIndexes = columnsToAlwaysShow.Select(col => Array.IndexOf(table[0], col));
int[] widths = new int[colCount];
bool[] areSame = new bool[colCount];
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
areSame[colIndex] = rowCount > 2 && colIndex < colCount - 3;
for (int rowIndex = 0; rowIndex < rowCount; rowIndex++)
{
widths[colIndex] = Math.Max(widths[colIndex], table[rowIndex][colIndex].Length + 1);
if (rowIndex > 1 && table[rowIndex][colIndex] != table[1][colIndex])
{
areSame[colIndex] = false;
}
}
}
if (areSame.Any(s => s))
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
if (areSame[colIndex] && columnsToShowIndexes.Contains(colIndex) == false)
{
logger.WriteInfo($"{table[0][colIndex]}={table[1][colIndex]} ");
}
}
logger.NewLine();
}
table.Insert(1, widths.Select(w => new string('-', w)).ToArray());
foreach (var row in table)
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
if (!areSame[colIndex] || columnsToShowIndexes.Contains(colIndex))
{
logger.WriteStatistic(row[colIndex].PadLeft(widths[colIndex], ' ') + " |");
}
}
logger.NewLine();
}
if (areSame.Any(s => s))
{
logger.NewLine();
}
}
private BenchmarkBuildResult Build(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain, BenchmarkGenerateResult generateResult)
{
logger.WriteLineInfo("// *** Build ***");
var buildResult = toolchain.Build(generateResult);
if (buildResult.IsBuildSuccess)
{
logger.WriteLineInfo("// Result = Success");
}
else
{
logger.WriteLineError("// Result = Failure");
if (buildResult.BuildException != null)
{
logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}");
}
}
logger.NewLine();
return(buildResult);
}
private BenchmarkGenerateResult Generate(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain)
{
logger.WriteLineInfo("// *** Generate *** ");
var generateResult = toolchain.Generate();
if (generateResult.IsGenerateSuccess)
{
logger.WriteLineInfo("// Result = Success");
logger.WriteLineInfo($"// {nameof(generateResult.DirectoryPath)} = {generateResult.DirectoryPath}");
}
else
{
logger.WriteLineError("// Result = Failure");
if (generateResult.GenerateException != null)
{
logger.WriteLineError($"// Exception: {generateResult.GenerateException.Message}");
}
}
logger.NewLine();
return(generateResult);
}
public void Export(IList <BenchmarkReport> reports, IBenchmarkLogger logger)
{
logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
var table = BenchmarkExporterHelper.BuildTable(reports);
// If we have Benchmarks with ParametersSets, force the "Method" columns to be displayed, otherwise it doesn't make as much sense
var columnsToAlwaysShow = reports.Any(r => r.Benchmark.Task.ParametersSets != null) ? new[] { "Method" } : new string[0];
PrintTable(table, logger, columnsToAlwaysShow);
var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
{
logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
}
}
private void CopyFile(string sourcePath, string destinationPath)
{
logger.WriteLineInfo("// Copying {0}", Path.GetFileName(sourcePath));
logger.WriteLineInfo("// from: {0}", Path.GetDirectoryName(sourcePath));
logger.WriteLineInfo("// to: {0}", Path.GetDirectoryName(destinationPath));
try
{
File.Copy(Path.GetFullPath(sourcePath), Path.GetFullPath(destinationPath), overwrite: true);
}
catch (Exception ex)
{
logger.WriteLineError(ex.Message);
throw;
}
}
private List <BenchmarkRunReport> Execute(IBenchmarkLogger logger, Benchmark benchmark, IList <string> importantPropertyNames, BenchmarkParameters parameters, IBenchmarkToolchainFacade toolchain, BenchmarkBuildResult buildResult)
{
logger.WriteLineInfo("// *** Execute ***");
var processCount = Math.Max(1, benchmark.Task.ProcessCount);
var runReports = new List <BenchmarkRunReport>();
for (int processNumber = 0; processNumber < processCount; processNumber++)
{
logger.WriteLineInfo($"// Run, Process: {processNumber + 1} / {processCount}");
if (parameters != null)
{
logger.WriteLineInfo($"// {parameters.ToInfo()}");
}
if (importantPropertyNames.Any())
{
logger.WriteInfo("// ");
foreach (var name in importantPropertyNames)
{
logger.WriteInfo($"{name}={benchmark.Properties.GetValue(name)} ");
}
logger.NewLine();
}
var execResult = toolchain.Execute(buildResult, parameters, Plugins.CompositeDiagnoser);
if (execResult.FoundExecutable)
{
var iterRunReports = execResult.Data.Select(line => BenchmarkRunReport.Parse(logger, line)).Where(r => r != null).ToList();
runReports.AddRange(iterRunReports);
}
else
{
logger.WriteLineError("Executable not found");
}
}
logger.NewLine();
return(runReports);
}
/// <summary>
/// Parses the benchmark statistics from the plain text line.
///
/// E.g. given the input <paramref name="line"/>:
///
/// Target 1: 10 op, 1005842518 ns
///
/// Will extract the number of <see cref="Operations"/> performed and the
/// total number of <see cref="Nanoseconds"/> it took to perform them.
/// </summary>
/// <param name="logger">The logger to write any diagnostic messages to.</param>
/// <param name="line">The line to parse.</param>
/// <returns>An instance of <see cref="BenchmarkRunReport"/> if parsed successfully. <c>Null</c> in case of any trouble.</returns>
public static BenchmarkRunReport Parse(IBenchmarkLogger logger, string line)
{
try
{
var lineSplit = line.Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
var iterationInfo = lineSplit[0];
var iterationInfoSplit = iterationInfo.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
var iterationMode = ParseIterationMode(iterationInfoSplit[0]);
var iterationIndex = 0;
int.TryParse(iterationInfoSplit[1], out iterationIndex);
var measurementsInfo = lineSplit[1];
var measurementsInfoSplit = measurementsInfo.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
var op = 1L;
var ns = double.PositiveInfinity;
foreach (var item in measurementsInfoSplit)
{
var measurementSplit = item.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
var value = measurementSplit[0];
var unit = measurementSplit[1];
switch (unit)
{
case "ns":
ns = double.Parse(value, EnvironmentInfo.MainCultureInfo);
break;
case "op":
op = long.Parse(value);
break;
}
}
return new BenchmarkRunReport(iterationMode, iterationIndex, op, ns);
}
catch (Exception)
{
logger.WriteLineError("Parse error in the following line:");
logger.WriteLineError(line);
return null;
}
}
private List<BenchmarkRunReport> Execute(IBenchmarkLogger logger, Benchmark benchmark, IList<string> importantPropertyNames, BenchmarkParameters parameters, IBenchmarkToolchainFacade toolchain, BenchmarkBuildResult buildResult)
{
logger.WriteLineInfo("// *** Execute ***");
var processCount = Math.Max(1, benchmark.Task.ProcessCount);
var runReports = new List<BenchmarkRunReport>();
for (int processNumber = 0; processNumber < processCount; processNumber++)
{
logger.WriteLineInfo($"// Run, Process: {processNumber + 1} / {processCount}");
if (parameters != null)
logger.WriteLineInfo($"// {parameters.ToInfo()}");
if (importantPropertyNames.Any())
{
logger.WriteInfo("// ");
foreach (var name in importantPropertyNames)
logger.WriteInfo($"{name}={benchmark.Properties.GetValue(name)} ");
logger.NewLine();
}
var execResult = toolchain.Execute(buildResult, parameters, Plugins.CompositeDiagnoser);
if (execResult.FoundExecutable)
{
var iterRunReports = execResult.Data.Select(line => BenchmarkRunReport.Parse(logger, line)).Where(r => r != null).ToList();
runReports.AddRange(iterRunReports);
}
else
{
logger.WriteLineError("Executable not found");
}
}
logger.NewLine();
return runReports;
}
private BenchmarkBuildResult Build(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain, BenchmarkGenerateResult generateResult)
{
logger.WriteLineInfo("// *** Build ***");
var buildResult = toolchain.Build(generateResult);
if (buildResult.IsBuildSuccess)
{
logger.WriteLineInfo("// Result = Success");
}
else
{
logger.WriteLineError("// Result = Failure");
if (buildResult.BuildException != null)
logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}");
}
logger.NewLine();
return buildResult;
}
private BenchmarkGenerateResult Generate(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain)
{
logger.WriteLineInfo("// *** Generate *** ");
var generateResult = toolchain.Generate();
if (generateResult.IsGenerateSuccess)
{
logger.WriteLineInfo("// Result = Success");
logger.WriteLineInfo($"// {nameof(generateResult.DirectoryPath)} = {generateResult.DirectoryPath}");
}
else
{
logger.WriteLineError("// Result = Failure");
if (generateResult.GenerateException != null)
logger.WriteLineError($"// Exception: {generateResult.GenerateException.Message}");
}
logger.NewLine();
return generateResult;
}
private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
logger.WriteLineInfo($"// {benchmark.Description}");
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
}
logger.NewLine();
}
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
BenchmarkMarkdownExporter.Default.Export(reports, logger);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// *** Warnings *** ");
foreach (var warning in warnings)
logger.WriteLineError($"{warning.Message}");
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return reports;
}
private List <BenchmarkReport> Run(List <Benchmark> benchmarks, IBenchmarkLogger logger)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
{
logger.WriteLineInfo($"// {benchmark.Description}");
}
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List <BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
}
logger.NewLine();
}
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
BenchmarkMarkdownExporter.Default.Export(reports, logger);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// *** Warnings *** ");
foreach (var warning in warnings)
{
logger.WriteLineError($"{warning.Message}");
}
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return(reports);
}
private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger, string competitionName)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
logger.WriteLineInfo($"// {benchmark.Description}");
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var globalStopwatch = Stopwatch.StartNew();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.GetTargetRuns().Any())
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.GetTargetRuns().Any())
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
}
}
logger.NewLine();
}
globalStopwatch.Stop();
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
logger.WriteLineHeader("// * Export *");
var files = Plugins.CompositeExporter.ExportToFile(reports, competitionName, Plugins.ResultExtenders);
foreach (var file in files)
logger.WriteLineInfo($" {file}");
logger.NewLine();
logger.WriteLineHeader("// * Detailed results *");
foreach (var report in reports)
{
logger.WriteLineInfo(report.Benchmark.Description);
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
logger.NewLine();
}
logger.WriteLineStatistic($"Total time: {globalStopwatch.Elapsed.TotalHours:00}:{globalStopwatch.Elapsed:mm\\:ss}");
logger.NewLine();
logger.WriteLineHeader("// * Summary *");
BenchmarkMarkdownExporter.Default.Export(reports, logger, Plugins.ResultExtenders);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// * Warnings * ");
foreach (var warning in warnings)
logger.WriteLineError($"{warning.Message}");
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return reports;
}