public override void ExportToLog(Summary summary, ILogger logger)
{
logger.WriteLine($"<pre><code>{EnvironmentHelper.GetCurrentInfo().ToFormattedString("Host")}</code></pre>");
logger.WriteLine();
PrintTable(summary.Table, logger);
}
internal SummaryTable(Summary summary)
{
Summary = summary;
var configColumns = summary.Config.
GetColumns().
Where(c => c.IsAvailable(summary));
var paramColumns = summary.Benchmarks.
SelectMany(b => b.Parameters.Items.Select(item => item.Name)).
Distinct().
Select(name => new ParamColumn(name));
var columns = configColumns.Concat(paramColumns).ToArray();
ColumnCount = columns.Length;
FullHeader = columns.Select(c => c.ColumnName).ToArray();
var reports = summary.Reports.Values.
OrderBy(r => r.Benchmark.Parameters.FullInfo).
ThenBy(r => r.Benchmark.Target.Type.Name).
ToList();
FullContent = reports.Select(r => columns.Select(c => c.GetValue(summary, r.Benchmark)).ToArray()).ToArray();
var full = new List<string[]> { FullHeader };
full.AddRange(FullContent);
FullContentWithHeader = full.ToArray();
Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i].AlwaysShow)).ToArray();
}
public override void ExportToLog(Summary summary, ILogger logger)
{
logger.WriteLine(string.Join(";", columns.Select(c => c.Title)));
foreach (var report in summary.Reports.Values)
foreach (var measurement in report.AllMeasurements)
logger.WriteLine(string.Join(";", columns.Select(column => column.GetValue(summary, report, measurement))));
}
public IEnumerable<IColumn> GetColumns(Summary summary)
{
return summary.Benchmarks.
SelectMany(b => b.Parameters.Items.Select(item => item.Name)).
Distinct().
Select(name => new ParamColumn(name));
}
public override void ExportToLog(Summary summary, ILogger logger)
{
logger.WriteLine("<!DOCTYPE html>");
logger.WriteLine("<html lang='en'>");
logger.WriteLine("<head>");
logger.WriteLine("<meta charset='utf-8' />");
logger.WriteLine("<title>" + summary.Title + "</title>");
logger.WriteLine(CssDefinition);
logger.WriteLine("</head>");
logger.WriteLine("<body>");
logger.Write("<pre><code>");
logger.WriteLine();
foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString())
{
logger.WriteLine(infoLine);
}
logger.WriteLine(summary.JobRuntimes);
logger.Write("</code></pre>");
logger.WriteLine();
PrintTable(summary.Table, logger);
logger.WriteLine("</body>");
logger.WriteLine("</html>");
}
public override void ExportToLog(Summary summary, ILogger logger)
{
if (useCodeBlocks)
{
logger.WriteLine(codeBlockStart);
}
logger = GetRightLogger(logger);
logger.WriteLine();
foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString())
{
logger.WriteLineInfo(infoLine);
}
logger.WriteLineInfo(summary.JobRuntimes);
logger.WriteLine();
PrintTable(summary.Table, logger);
// TODO: move this logic to an analyser
var benchmarksWithTroubles = summary.Reports.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.WriteLine();
logger.WriteLineError("Benchmarks with issues:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
{
logger.WriteLineError(" " + benchmarkWithTroubles.DisplayInfo);
}
}
}
public string GetValue(Summary summary, Benchmark benchmark)
{
var ranks = RankHelper.GetRanks(summary.Reports.Select(r => r.ResultStatistics).ToArray());
int index = Array.IndexOf(summary.Reports.Select(r => r.Benchmark).ToArray(), benchmark);
int rank = ranks[index];
return system.ToPresentation(rank);
}
public IEnumerable<string> ExportToFiles(Summary summary)
{
var fileName = $"{Path.Combine(summary.CurrentDirectory, summary.Title)}-{FileCaption}{FileNameSuffix}.{FileExtension}";
using (var stream = new StreamWriter(fileName))
ExportToLog(summary, new StreamLogger(stream));
yield return fileName;
}
public override void ExportToLog(Summary summary, ILogger logger)
{
logger.WriteLine("....");
foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString())
{
logger.WriteLineInfo(infoLine);
}
logger.WriteLineInfo(summary.JobRuntimes);
logger.WriteLine();
PrintTable(summary.Table, logger);
var benchmarksWithTroubles = summary.Reports
.Where(r => !r.GetResultRuns().Any())
.Select(r => r.Benchmark)
.ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.WriteLine();
logger.WriteLine("[WARNING]");
logger.WriteLineError(".Benchmarks with issues");
logger.WriteLine("====");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
logger.WriteLineError("* " + benchmarkWithTroubles.DisplayInfo);
logger.WriteLine("====");
}
}
public IEnumerable<string> ExportToFiles(Summary summary)
{
var fileNames = new List<string>();
foreach (var exporter in exporters)
fileNames.AddRange(exporter.ExportToFiles(summary));
return fileNames;
}
public string GetValue(Summary summary, Benchmark benchmark)
{
var baselineBenchmark = summary.Benchmarks.
Where(b => b.Job.GetFullInfo() == benchmark.Job.GetFullInfo()).
Where(b => b.Parameters.FullInfo == benchmark.Parameters.FullInfo).
FirstOrDefault(b => b.Target.Baseline);
if (baselineBenchmark == null)
return "?";
var baselineMedian = summary.Reports[baselineBenchmark].ResultStatistics.Median;
var currentMedian = summary.Reports[benchmark].ResultStatistics.Median;
switch (Kind)
{
case DiffKind.Delta:
if (benchmark.Target.Baseline)
return "Baseline";
var diff = (currentMedian - baselineMedian)/baselineMedian*100.0;
return diff.ToStr("0.0") + "%";
case DiffKind.Scaled:
var scale = currentMedian/baselineMedian;
return scale.ToStr("0.00");
default:
return "?";
}
}
public IEnumerable<string> ExportToFiles(Summary summary)
{
const string scriptFileName = "BuildPlots.R";
yield return scriptFileName;
var fileNamePrefix = Path.Combine(summary.ResultsDirectoryPath, summary.Title);
var scriptFullPath = Path.Combine(summary.ResultsDirectoryPath, scriptFileName);
var script = ResourceHelper.LoadTemplate(scriptFileName).Replace("$BenchmarkDotNetVersion$", BenchmarkDotNetInfo.FullTitle);
lock (buildScriptLock)
File.WriteAllText(scriptFullPath, script);
var rHome = Environment.GetEnvironmentVariable("R_HOME") ?? @"C:\Program Files\R\R-3.2.3\bin\";
if (Directory.Exists(rHome))
{
var start = new ProcessStartInfo
{
UseShellExecute = false,
RedirectStandardOutput = false,
CreateNoWindow = true,
FileName = Path.Combine(rHome, "Rscript.exe"),
WorkingDirectory = summary.ResultsDirectoryPath,
Arguments = $"\"{scriptFullPath}\" \"{fileNamePrefix}-measurements.csv\""
};
using (var process = Process.Start(start))
process?.WaitForExit();
yield return fileNamePrefix + "-boxplot.png";
yield return fileNamePrefix + "-barplot.png";
}
}
public IEnumerable<IWarning> Analyze(Summary summary)
{
var hostInfo = summary.HostEnvironmentHelper;
if (hostInfo.HasAttachedDebugger)
yield return new Warning("AttachedDebugger", "Benchmark was executed with attached debugger.", null);
if (hostInfo.Configuration.Equals("DEBUG", StringComparison.InvariantCultureIgnoreCase))
yield return new Warning("DebugConfiguration", "Benchmark was built in DEBUG configuration. Please, build it in RELEASE.", null);
}
public IEnumerable<IWarning> Analyse(Summary summary)
{
var hostInfo = summary.HostEnvironmentInfo;
if (hostInfo.HasAttachedDebugger)
yield return new Warning("AttachedDebugger", "Benchmark was executed with attached debugger.", null);
if (hostInfo.Configuration.EqualsWithIgnoreCase("DEBUG"))
yield return new Warning("DebugConfiguration", "Benchmark was built in DEBUG configuration. Please, build it in RELEASE.", null);
}
public IEnumerable<IColumn> GetColumns(Summary summary)
{
if (summary.Benchmarks.Select(b => b.Target.Type.Namespace).Distinct().Count() > 1)
yield return TargetMethodColumn.Namespace;
if (summary.Benchmarks.Select(b => b.Target.Type.Name).Distinct().Count() > 1)
yield return TargetMethodColumn.Type;
yield return TargetMethodColumn.Method;
}
public IEnumerable<Conclusion> Analyse(Summary summary)
{
foreach (var conclusion in AnalyseSummary(summary))
yield return conclusion;
foreach (var report in summary.Reports)
foreach (var conclusion in AnalyseReport(report))
yield return conclusion;
}
public IEnumerable<string> ExportToFiles(Summary summary)
{
var filePath = $"{Path.Combine(summary.ResultsDirectoryPath, summary.Title)}-{FileCaption}{FileNameSuffix}.{FileExtension}";
using (var stream = Portability.StreamWriter.FromPath(filePath))
{
ExportToLog(summary, new StreamLogger(stream));
}
return new [] { filePath };
}
public override void ExportToLog(Summary summary, ILogger logger)
{
logger.Write("<pre><code>");
logger.WriteLine();
foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString())
{
logger.WriteLine(infoLine);
}
logger.Write("</code></pre>");
logger.WriteLine();
PrintTable(summary.Table, logger);
}
internal SummaryTable(Summary summary)
{
Summary = summary;
if (summary.HasCriticalValidationErrors)
{
Columns = new SummaryTableColumn[0];
ColumnCount = 0;
FullHeader = new string[0];
FullContent = new string[0][];
FullContentStartOfGroup = new bool[0];
FullContentWithHeader = new string[0][];
return;
}
var configColumns = summary.Config.
GetColumns().
Where(c => c.IsAvailable(summary));
var paramColumns = summary.Benchmarks.
SelectMany(b => b.Parameters.Items.Select(item => item.Name)).
Distinct().
Select(name => new ParamColumn(name));
var diagnoserColumns = summary.Config.
GetDiagnosers().
Where(d => d is IColumnProvider).
Cast<IColumnProvider>().
SelectMany(cp => cp.GetColumns);
var columns = configColumns.Concat(paramColumns).Concat(diagnoserColumns).ToArray().OrderBy(c => c.Category).ToArray();
ColumnCount = columns.Length;
FullHeader = columns.Select(c => c.ColumnName).ToArray();
var orderProvider = summary.Config.GetOrderProvider() ?? DefaultOrderProvider.Instance;
FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.Benchmark)).ToArray()).ToArray();
var groupKeys = summary.Benchmarks.Select(b => orderProvider.GetGroupKey(b, summary)).ToArray();
FullContentStartOfGroup = new bool[summary.Reports.Length];
if (groupKeys.Distinct().Count() > 1 && FullContentStartOfGroup.Length > 0)
{
FullContentStartOfGroup[0] = true;
for (int i = 1; i < summary.Reports.Length; i++)
FullContentStartOfGroup[i] = groupKeys[i] != groupKeys[i - 1];
}
var full = new List<string[]> { FullHeader };
full.AddRange(FullContent);
FullContentWithHeader = full.ToArray();
Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i].AlwaysShow)).ToArray();
}
public IEnumerable<IWarning> Analyse(Summary summary)
{
if (summary.HostEnvironmentInfo.HasAttachedDebugger)
yield return new Warning("AttachedDebugger", "Benchmark was executed with attached debugger.", null);
foreach (var benchmark in summary.Benchmarks
.Where(benchmark => benchmark.Target.Type.GetTypeInfo().Assembly.IsDebug().IsTrue()))
{
yield return
new Warning(
"DebugConfiguration",
"Benchmark was built in DEBUG configuration. Please, build it in RELEASE.",
summary.Reports.FirstOrDefault(report => ReferenceEquals(report.Benchmark, benchmark)));
}
}
public string GetValue(Summary summary, Benchmark benchmark)
{
var baselineBenchmark = summary.Benchmarks.
Where(b => b.Job.GetFullInfo() == benchmark.Job.GetFullInfo()).
Where(b => b.Parameters.FullInfo == benchmark.Parameters.FullInfo).
FirstOrDefault(b => b.Target.Baseline);
var invalidResults = baselineBenchmark == null ||
summary[baselineBenchmark] == null ||
summary[baselineBenchmark].ResultStatistics == null ||
summary[benchmark] == null ||
summary[benchmark].ResultStatistics == null;
if (invalidResults)
return "?";
double baselineMetric;
double currentMetric;
var resultStatistics = summary[baselineBenchmark].ResultStatistics;
var statistics = summary[benchmark].ResultStatistics;
if (Percentile == null)
{
baselineMetric = resultStatistics.Median;
currentMetric = statistics.Median;
}
else
{
baselineMetric = resultStatistics.Percentiles.Percentile(Percentile.GetValueOrDefault());
currentMetric = statistics.Percentiles.Percentile(Percentile.GetValueOrDefault());
}
if (baselineMetric == 0)
return "?";
switch (Kind)
{
case DiffKind.Delta:
if (benchmark.Target.Baseline)
return "Baseline";
var diff = (currentMetric - baselineMetric) / baselineMetric * 100.0;
return diff.ToStr("N1") + "%";
case DiffKind.Scaled:
var scale = currentMetric / baselineMetric;
return scale.ToStr("N2");
default:
return "?";
}
}
public string GetValue(Summary summary, Benchmark benchmark)
{
var sortedBenchmarks = summary.Benchmarks.
OrderBy(b => summary.Reports[b].ResultStatistics.Mean).
ToArray();
var places = GetPlaces(sortedBenchmarks.Select(b => summary.Reports[b].ResultStatistics).ToArray());
var place = places[Array.IndexOf(sortedBenchmarks, benchmark)];
switch (kind)
{
case Kind.ArabicNumber:
return place.ToString();
case Kind.Stars:
return new string('*', place);
}
return place.ToString();
}
public IEnumerable<IColumn> GetColumns(Summary summary)
{
yield return StatisticColumn.Mean;
if (NeedToShow(summary, s => s.StandardError > s.Mean * 0.01)) // TODO: Use Accuracy.MaxStdErrRelative
yield return StatisticColumn.StdErr;
if (NeedToShow(summary, s => s.Percentiles.P95 > s.Mean + 3 * s.StandardDeviation))
yield return StatisticColumn.P95;
if (NeedToShow(summary, s=> s.GetConfidenceInterval(ConfidenceLevel.L99).Contains(s.Median)))
yield return StatisticColumn.Median;
if (NeedToShow(summary, s => s.StandardDeviation > 1e-9))
yield return StatisticColumn.StdDev;
yield return BaselineScaledColumn.Scaled;
yield return BaselineScaledColumn.ScaledStdDev; // TODO: make optional
}
public override void ExportToLog(Summary summary, ILogger logger)
{
foreach (var line in summary.Table.FullContentWithHeader)
{
for (int i = 0; i < line.Length;)
{
logger.Write(CsvHelper.Escape(line[i]));
if (++i < line.Length)
{
logger.Write(separator);
}
}
logger.WriteLine();
}
}
public override void ExportToLog(Summary summary, ILogger logger)
{
foreach (var report in summary.Reports.Values)
{
var runs = report.AllMeasurements;
var modes = runs.Select(it => it.IterationMode).Distinct();
logger.WriteLineHeader($"*** {report.Benchmark.ShortInfo} ***");
logger.WriteLineHeader("* Raw *");
foreach (var run in runs)
logger.WriteLineResult(run.ToStr());
foreach (var mode in modes)
{
logger.NewLine();
logger.WriteLineHeader($"* Statistics for {mode}");
logger.WriteLineStatistic(runs.Where(it => it.IterationMode == mode).GetStatistics().ToTimeStr());
}
}
}
public override void ExportToLog(Summary summary, ILogger logger)
{
if (useCodeBlocks)
logger.WriteLine($"```{codeBlocksSyntax}");
logger = new LoggerWithPrefix(logger, prefix);
logger.WriteLineInfo(EnvironmentHelper.GetCurrentInfo().ToFormattedString("Host"));
logger.NewLine();
PrintTable(summary.Table, logger);
// TODO: move this logic to an analyser
var benchmarksWithTroubles = summary.Reports.Values.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
logger.WriteLineError(" " + benchmarkWithTroubles.ShortInfo);
}
}
public override void ExportToLog(Summary summary, ILogger logger)
{
logger.WriteLine(string.Join(";", columns.Select(c => c.Title)));
foreach (var report in summary.Reports)
{
foreach (var measurement in report.AllMeasurements)
{
for (int i = 0; i < columns.Length; )
{
logger.Write(columns[i].GetValue(summary, report, measurement));
if (++i < columns.Length)
{
logger.Write(";");
}
}
logger.WriteLine();
}
}
}
internal SummaryTable(Summary summary)
{
Summary = summary;
if (summary.HasCriticalValidationErrors)
{
Columns = new SummaryTableColumn[0];
ColumnCount = 0;
FullHeader = new string[0];
FullContent = new string[0][];
FullContentStartOfGroup = new bool[0];
FullContentWithHeader = new string[0][];
IsDefault = new bool[0];
return;
}
var columns = summary.GetColumns();
ColumnCount = columns.Length;
FullHeader = columns.Select(c => c.ColumnName).ToArray();
var orderProvider = summary.Config.GetOrderProvider() ?? DefaultOrderProvider.Instance;
FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.Benchmark)).ToArray()).ToArray();
IsDefault = columns.Select(c => summary.Reports.All(r => c.IsDefault(summary, r.Benchmark))).ToArray();
var groupKeys = summary.Benchmarks.Select(b => orderProvider.GetGroupKey(b, summary)).ToArray();
FullContentStartOfGroup = new bool[summary.Reports.Length];
if (groupKeys.Distinct().Count() > 1 && FullContentStartOfGroup.Length > 0)
{
FullContentStartOfGroup[0] = true;
for (int i = 1; i < summary.Reports.Length; i++)
FullContentStartOfGroup[i] = groupKeys[i] != groupKeys[i - 1];
}
var full = new List<string[]> { FullHeader };
full.AddRange(FullContent);
FullContentWithHeader = full.ToArray();
Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i].AlwaysShow)).ToArray();
}
public override void ExportToLog(Summary summary, ILogger logger)
{
var columns = lazyColumns.Value;
logger.WriteLine(string.Join(Separator, columns.Select(c => CsvHelper.Escape(c.Title))));
foreach (var report in summary.Reports)
{
foreach (var measurement in report.AllMeasurements)
{
for (int i = 0; i < columns.Length; )
{
logger.Write(CsvHelper.Escape(columns[i].GetValue(summary, report, measurement)));
if (++i < columns.Length)
{
logger.Write(Separator);
}
}
logger.WriteLine();
}
}
}
public string GetValue(Summary summary, Benchmark benchmark)
{
var baseline = summary.Benchmarks.
Where(b => b.Job.DisplayInfo == benchmark.Job.DisplayInfo).
Where(b => b.Parameters.DisplayInfo == benchmark.Parameters.DisplayInfo).
FirstOrDefault(b => b.Target.Baseline);
bool invalidResults = baseline == null ||
summary[baseline] == null ||
summary[baseline].ResultStatistics == null ||
!summary[baseline].ResultStatistics.CanBeInverted() ||
summary[benchmark] == null ||
summary[benchmark].ResultStatistics == null;
if (invalidResults)
return "?";
var baselineStat = summary[baseline].ResultStatistics;
var targetStat = summary[benchmark].ResultStatistics;
double mean = benchmark.Target.Baseline ? 1 : Statistics.DivMean(targetStat, baselineStat);
double stdDev = benchmark.Target.Baseline ? 0 : Math.Sqrt(Statistics.DivVariance(targetStat, baselineStat));
switch (Kind)
{
case DiffKind.Mean:
return mean.ToStr("N2");
case DiffKind.StdDev:
return stdDev.ToStr("N2");
case DiffKind.WelchTTestPValue:
{
double pvalue = WelchTTest.Calc(baselineStat, targetStat).PValue;
return pvalue > 0.0001 ? pvalue.ToStr("N4") : pvalue.ToStr("e2");
}
default:
throw new NotSupportedException();
}
}
internal SummaryTable(Summary summary, SummaryStyle style = null)
{
Summary = summary;
if (summary.HasCriticalValidationErrors)
{
Columns = Array.Empty <SummaryTableColumn>();
ColumnCount = 0;
FullHeader = Array.Empty <string>();
FullContent = Array.Empty <string[]>();
FullContentStartOfHighlightGroup = Array.Empty <bool>();
FullContentWithHeader = Array.Empty <string[]>();
IsDefault = Array.Empty <bool>();
return;
}
// Ensure we have all required data for styling
style = style ?? SummaryStyle.Default;
if (style.TimeUnit == null)
{
style = style.WithTimeUnit(TimeUnit.GetBestTimeUnit(summary.Reports.Where(r => r.ResultStatistics != null).Select(r => r.ResultStatistics.Mean)
.ToArray()));
}
if (style.SizeUnit == null)
{
style = style.WithSizeUnit(SizeUnit.GetBestSizeUnit(summary.Reports.Select(r => r.GcStats.BytesAllocatedPerOperation).ToArray()));
}
var columns = summary.GetColumns();
ColumnCount = columns.Length;
FullHeader = columns.Select(c => c.GetColumnTitle(style)).ToArray();
FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.BenchmarkCase, style)).ToArray()).ToArray();
IsDefault = columns.Select(c => summary.Reports.All(r => c.IsDefault(summary, r.BenchmarkCase))).ToArray();
var highlightGroupKeys = summary.BenchmarksCases.Select(b => b.Config.Orderer.GetHighlightGroupKey(b)).ToArray();
FullContentStartOfHighlightGroup = new bool[summary.Reports.Length];
if (highlightGroupKeys.Distinct().Count() > 1 && FullContentStartOfHighlightGroup.Length > 0)
{
FullContentStartOfHighlightGroup[0] = true;
for (int i = 1; i < summary.Reports.Length; i++)
{
FullContentStartOfHighlightGroup[i] = highlightGroupKeys[i] != highlightGroupKeys[i - 1];
}
}
var logicalGroupKeys = summary.BenchmarksCases
.Select(b => b.Config.Orderer.GetLogicalGroupKey(summary.BenchmarksCases, b))
.ToArray();
FullContentStartOfLogicalGroup = new bool[summary.Reports.Length];
if (logicalGroupKeys.Distinct().Count() > 1 && FullContentStartOfLogicalGroup.Length > 0)
{
FullContentStartOfLogicalGroup[0] = true;
for (int i = 1; i < summary.Reports.Length; i++)
{
FullContentStartOfLogicalGroup[i] = logicalGroupKeys[i] != logicalGroupKeys[i - 1];
}
}
SeparateLogicalGroups = summary.Orderer.SeparateLogicalGroups;
var full = new List <string[]> {
FullHeader
};
full.AddRange(FullContent);
FullContentWithHeader = full.ToArray();
Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i])).ToArray();
EffectiveSummaryStyle = style;
}
