public override void ExportToLog(Summary summary, ILogger logger) { logger.WriteLine($"<pre><code>{EnvironmentHelper.GetCurrentInfo().ToFormattedString("Host")}</code></pre>"); logger.WriteLine(); PrintTable(summary.Table, logger); }
internal SummaryTable(Summary summary) { Summary = summary; var configColumns = summary.Config. GetColumns(). Where(c => c.IsAvailable(summary)); var paramColumns = summary.Benchmarks. SelectMany(b => b.Parameters.Items.Select(item => item.Name)). Distinct(). Select(name => new ParamColumn(name)); var columns = configColumns.Concat(paramColumns).ToArray(); ColumnCount = columns.Length; FullHeader = columns.Select(c => c.ColumnName).ToArray(); var reports = summary.Reports.Values. OrderBy(r => r.Benchmark.Parameters.FullInfo). ThenBy(r => r.Benchmark.Target.Type.Name). ToList(); FullContent = reports.Select(r => columns.Select(c => c.GetValue(summary, r.Benchmark)).ToArray()).ToArray(); var full = new List<string[]> { FullHeader }; full.AddRange(FullContent); FullContentWithHeader = full.ToArray(); Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i].AlwaysShow)).ToArray(); }
public override void ExportToLog(Summary summary, ILogger logger) { logger.WriteLine(string.Join(";", columns.Select(c => c.Title))); foreach (var report in summary.Reports.Values) foreach (var measurement in report.AllMeasurements) logger.WriteLine(string.Join(";", columns.Select(column => column.GetValue(summary, report, measurement)))); }
public IEnumerable<IColumn> GetColumns(Summary summary) { return summary.Benchmarks. SelectMany(b => b.Parameters.Items.Select(item => item.Name)). Distinct(). Select(name => new ParamColumn(name)); }
public override void ExportToLog(Summary summary, ILogger logger) { logger.WriteLine("<!DOCTYPE html>"); logger.WriteLine("<html lang='en'>"); logger.WriteLine("<head>"); logger.WriteLine("<meta charset='utf-8' />"); logger.WriteLine("<title>" + summary.Title + "</title>"); logger.WriteLine(CssDefinition); logger.WriteLine("</head>"); logger.WriteLine("<body>"); logger.Write("<pre><code>"); logger.WriteLine(); foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString()) { logger.WriteLine(infoLine); } logger.WriteLine(summary.JobRuntimes); logger.Write("</code></pre>"); logger.WriteLine(); PrintTable(summary.Table, logger); logger.WriteLine("</body>"); logger.WriteLine("</html>"); }
public override void ExportToLog(Summary summary, ILogger logger) { if (useCodeBlocks) { logger.WriteLine(codeBlockStart); } logger = GetRightLogger(logger); logger.WriteLine(); foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString()) { logger.WriteLineInfo(infoLine); } logger.WriteLineInfo(summary.JobRuntimes); logger.WriteLine(); PrintTable(summary.Table, logger); // TODO: move this logic to an analyser var benchmarksWithTroubles = summary.Reports.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList(); if (benchmarksWithTroubles.Count > 0) { logger.WriteLine(); logger.WriteLineError("Benchmarks with issues:"); foreach (var benchmarkWithTroubles in benchmarksWithTroubles) { logger.WriteLineError(" " + benchmarkWithTroubles.DisplayInfo); } } }
public string GetValue(Summary summary, Benchmark benchmark) { var ranks = RankHelper.GetRanks(summary.Reports.Select(r => r.ResultStatistics).ToArray()); int index = Array.IndexOf(summary.Reports.Select(r => r.Benchmark).ToArray(), benchmark); int rank = ranks[index]; return system.ToPresentation(rank); }
public IEnumerable<string> ExportToFiles(Summary summary) { var fileName = $"{Path.Combine(summary.CurrentDirectory, summary.Title)}-{FileCaption}{FileNameSuffix}.{FileExtension}"; using (var stream = new StreamWriter(fileName)) ExportToLog(summary, new StreamLogger(stream)); yield return fileName; }
public override void ExportToLog(Summary summary, ILogger logger) { logger.WriteLine("...."); foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString()) { logger.WriteLineInfo(infoLine); } logger.WriteLineInfo(summary.JobRuntimes); logger.WriteLine(); PrintTable(summary.Table, logger); var benchmarksWithTroubles = summary.Reports .Where(r => !r.GetResultRuns().Any()) .Select(r => r.Benchmark) .ToList(); if (benchmarksWithTroubles.Count > 0) { logger.WriteLine(); logger.WriteLine("[WARNING]"); logger.WriteLineError(".Benchmarks with issues"); logger.WriteLine("===="); foreach (var benchmarkWithTroubles in benchmarksWithTroubles) logger.WriteLineError("* " + benchmarkWithTroubles.DisplayInfo); logger.WriteLine("===="); } }
public IEnumerable<string> ExportToFiles(Summary summary) { var fileNames = new List<string>(); foreach (var exporter in exporters) fileNames.AddRange(exporter.ExportToFiles(summary)); return fileNames; }
public string GetValue(Summary summary, Benchmark benchmark) { var baselineBenchmark = summary.Benchmarks. Where(b => b.Job.GetFullInfo() == benchmark.Job.GetFullInfo()). Where(b => b.Parameters.FullInfo == benchmark.Parameters.FullInfo). FirstOrDefault(b => b.Target.Baseline); if (baselineBenchmark == null) return "?"; var baselineMedian = summary.Reports[baselineBenchmark].ResultStatistics.Median; var currentMedian = summary.Reports[benchmark].ResultStatistics.Median; switch (Kind) { case DiffKind.Delta: if (benchmark.Target.Baseline) return "Baseline"; var diff = (currentMedian - baselineMedian)/baselineMedian*100.0; return diff.ToStr("0.0") + "%"; case DiffKind.Scaled: var scale = currentMedian/baselineMedian; return scale.ToStr("0.00"); default: return "?"; } }
public IEnumerable<string> ExportToFiles(Summary summary) { const string scriptFileName = "BuildPlots.R"; yield return scriptFileName; var fileNamePrefix = Path.Combine(summary.ResultsDirectoryPath, summary.Title); var scriptFullPath = Path.Combine(summary.ResultsDirectoryPath, scriptFileName); var script = ResourceHelper.LoadTemplate(scriptFileName).Replace("$BenchmarkDotNetVersion$", BenchmarkDotNetInfo.FullTitle); lock (buildScriptLock) File.WriteAllText(scriptFullPath, script); var rHome = Environment.GetEnvironmentVariable("R_HOME") ?? @"C:\Program Files\R\R-3.2.3\bin\"; if (Directory.Exists(rHome)) { var start = new ProcessStartInfo { UseShellExecute = false, RedirectStandardOutput = false, CreateNoWindow = true, FileName = Path.Combine(rHome, "Rscript.exe"), WorkingDirectory = summary.ResultsDirectoryPath, Arguments = $"\"{scriptFullPath}\" \"{fileNamePrefix}-measurements.csv\"" }; using (var process = Process.Start(start)) process?.WaitForExit(); yield return fileNamePrefix + "-boxplot.png"; yield return fileNamePrefix + "-barplot.png"; } }
public IEnumerable<IWarning> Analyze(Summary summary) { var hostInfo = summary.HostEnvironmentHelper; if (hostInfo.HasAttachedDebugger) yield return new Warning("AttachedDebugger", "Benchmark was executed with attached debugger.", null); if (hostInfo.Configuration.Equals("DEBUG", StringComparison.InvariantCultureIgnoreCase)) yield return new Warning("DebugConfiguration", "Benchmark was built in DEBUG configuration. Please, build it in RELEASE.", null); }
public IEnumerable<IWarning> Analyse(Summary summary) { var hostInfo = summary.HostEnvironmentInfo; if (hostInfo.HasAttachedDebugger) yield return new Warning("AttachedDebugger", "Benchmark was executed with attached debugger.", null); if (hostInfo.Configuration.EqualsWithIgnoreCase("DEBUG")) yield return new Warning("DebugConfiguration", "Benchmark was built in DEBUG configuration. Please, build it in RELEASE.", null); }
public IEnumerable<IColumn> GetColumns(Summary summary) { if (summary.Benchmarks.Select(b => b.Target.Type.Namespace).Distinct().Count() > 1) yield return TargetMethodColumn.Namespace; if (summary.Benchmarks.Select(b => b.Target.Type.Name).Distinct().Count() > 1) yield return TargetMethodColumn.Type; yield return TargetMethodColumn.Method; }
public IEnumerable<Conclusion> Analyse(Summary summary) { foreach (var conclusion in AnalyseSummary(summary)) yield return conclusion; foreach (var report in summary.Reports) foreach (var conclusion in AnalyseReport(report)) yield return conclusion; }
public IEnumerable<string> ExportToFiles(Summary summary) { var filePath = $"{Path.Combine(summary.ResultsDirectoryPath, summary.Title)}-{FileCaption}{FileNameSuffix}.{FileExtension}"; using (var stream = Portability.StreamWriter.FromPath(filePath)) { ExportToLog(summary, new StreamLogger(stream)); } return new [] { filePath }; }
public override void ExportToLog(Summary summary, ILogger logger) { logger.Write("<pre><code>"); logger.WriteLine(); foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString()) { logger.WriteLine(infoLine); } logger.Write("</code></pre>"); logger.WriteLine(); PrintTable(summary.Table, logger); }
internal SummaryTable(Summary summary) { Summary = summary; if (summary.HasCriticalValidationErrors) { Columns = new SummaryTableColumn[0]; ColumnCount = 0; FullHeader = new string[0]; FullContent = new string[0][]; FullContentStartOfGroup = new bool[0]; FullContentWithHeader = new string[0][]; return; } var configColumns = summary.Config. GetColumns(). Where(c => c.IsAvailable(summary)); var paramColumns = summary.Benchmarks. SelectMany(b => b.Parameters.Items.Select(item => item.Name)). Distinct(). Select(name => new ParamColumn(name)); var diagnoserColumns = summary.Config. GetDiagnosers(). Where(d => d is IColumnProvider). Cast<IColumnProvider>(). SelectMany(cp => cp.GetColumns); var columns = configColumns.Concat(paramColumns).Concat(diagnoserColumns).ToArray().OrderBy(c => c.Category).ToArray(); ColumnCount = columns.Length; FullHeader = columns.Select(c => c.ColumnName).ToArray(); var orderProvider = summary.Config.GetOrderProvider() ?? DefaultOrderProvider.Instance; FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.Benchmark)).ToArray()).ToArray(); var groupKeys = summary.Benchmarks.Select(b => orderProvider.GetGroupKey(b, summary)).ToArray(); FullContentStartOfGroup = new bool[summary.Reports.Length]; if (groupKeys.Distinct().Count() > 1 && FullContentStartOfGroup.Length > 0) { FullContentStartOfGroup[0] = true; for (int i = 1; i < summary.Reports.Length; i++) FullContentStartOfGroup[i] = groupKeys[i] != groupKeys[i - 1]; } var full = new List<string[]> { FullHeader }; full.AddRange(FullContent); FullContentWithHeader = full.ToArray(); Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i].AlwaysShow)).ToArray(); }
public IEnumerable<IWarning> Analyse(Summary summary) { if (summary.HostEnvironmentInfo.HasAttachedDebugger) yield return new Warning("AttachedDebugger", "Benchmark was executed with attached debugger.", null); foreach (var benchmark in summary.Benchmarks .Where(benchmark => benchmark.Target.Type.GetTypeInfo().Assembly.IsDebug().IsTrue())) { yield return new Warning( "DebugConfiguration", "Benchmark was built in DEBUG configuration. Please, build it in RELEASE.", summary.Reports.FirstOrDefault(report => ReferenceEquals(report.Benchmark, benchmark))); } }
public string GetValue(Summary summary, Benchmark benchmark) { var baselineBenchmark = summary.Benchmarks. Where(b => b.Job.GetFullInfo() == benchmark.Job.GetFullInfo()). Where(b => b.Parameters.FullInfo == benchmark.Parameters.FullInfo). FirstOrDefault(b => b.Target.Baseline); var invalidResults = baselineBenchmark == null || summary[baselineBenchmark] == null || summary[baselineBenchmark].ResultStatistics == null || summary[benchmark] == null || summary[benchmark].ResultStatistics == null; if (invalidResults) return "?"; double baselineMetric; double currentMetric; var resultStatistics = summary[baselineBenchmark].ResultStatistics; var statistics = summary[benchmark].ResultStatistics; if (Percentile == null) { baselineMetric = resultStatistics.Median; currentMetric = statistics.Median; } else { baselineMetric = resultStatistics.Percentiles.Percentile(Percentile.GetValueOrDefault()); currentMetric = statistics.Percentiles.Percentile(Percentile.GetValueOrDefault()); } if (baselineMetric == 0) return "?"; switch (Kind) { case DiffKind.Delta: if (benchmark.Target.Baseline) return "Baseline"; var diff = (currentMetric - baselineMetric) / baselineMetric * 100.0; return diff.ToStr("N1") + "%"; case DiffKind.Scaled: var scale = currentMetric / baselineMetric; return scale.ToStr("N2"); default: return "?"; } }
public string GetValue(Summary summary, Benchmark benchmark) { var sortedBenchmarks = summary.Benchmarks. OrderBy(b => summary.Reports[b].ResultStatistics.Mean). ToArray(); var places = GetPlaces(sortedBenchmarks.Select(b => summary.Reports[b].ResultStatistics).ToArray()); var place = places[Array.IndexOf(sortedBenchmarks, benchmark)]; switch (kind) { case Kind.ArabicNumber: return place.ToString(); case Kind.Stars: return new string('*', place); } return place.ToString(); }
public IEnumerable<IColumn> GetColumns(Summary summary) { yield return StatisticColumn.Mean; if (NeedToShow(summary, s => s.StandardError > s.Mean * 0.01)) // TODO: Use Accuracy.MaxStdErrRelative yield return StatisticColumn.StdErr; if (NeedToShow(summary, s => s.Percentiles.P95 > s.Mean + 3 * s.StandardDeviation)) yield return StatisticColumn.P95; if (NeedToShow(summary, s=> s.GetConfidenceInterval(ConfidenceLevel.L99).Contains(s.Median))) yield return StatisticColumn.Median; if (NeedToShow(summary, s => s.StandardDeviation > 1e-9)) yield return StatisticColumn.StdDev; yield return BaselineScaledColumn.Scaled; yield return BaselineScaledColumn.ScaledStdDev; // TODO: make optional }
public override void ExportToLog(Summary summary, ILogger logger) { foreach (var line in summary.Table.FullContentWithHeader) { for (int i = 0; i < line.Length;) { logger.Write(CsvHelper.Escape(line[i])); if (++i < line.Length) { logger.Write(separator); } } logger.WriteLine(); } }
public override void ExportToLog(Summary summary, ILogger logger) { foreach (var report in summary.Reports.Values) { var runs = report.AllMeasurements; var modes = runs.Select(it => it.IterationMode).Distinct(); logger.WriteLineHeader($"*** {report.Benchmark.ShortInfo} ***"); logger.WriteLineHeader("* Raw *"); foreach (var run in runs) logger.WriteLineResult(run.ToStr()); foreach (var mode in modes) { logger.NewLine(); logger.WriteLineHeader($"* Statistics for {mode}"); logger.WriteLineStatistic(runs.Where(it => it.IterationMode == mode).GetStatistics().ToTimeStr()); } } }
public override void ExportToLog(Summary summary, ILogger logger) { if (useCodeBlocks) logger.WriteLine($"```{codeBlocksSyntax}"); logger = new LoggerWithPrefix(logger, prefix); logger.WriteLineInfo(EnvironmentHelper.GetCurrentInfo().ToFormattedString("Host")); logger.NewLine(); PrintTable(summary.Table, logger); // TODO: move this logic to an analyser var benchmarksWithTroubles = summary.Reports.Values.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList(); if (benchmarksWithTroubles.Count > 0) { logger.NewLine(); logger.WriteLineError("Benchmarks with troubles:"); foreach (var benchmarkWithTroubles in benchmarksWithTroubles) logger.WriteLineError(" " + benchmarkWithTroubles.ShortInfo); } }
public override void ExportToLog(Summary summary, ILogger logger) { logger.WriteLine(string.Join(";", columns.Select(c => c.Title))); foreach (var report in summary.Reports) { foreach (var measurement in report.AllMeasurements) { for (int i = 0; i < columns.Length; ) { logger.Write(columns[i].GetValue(summary, report, measurement)); if (++i < columns.Length) { logger.Write(";"); } } logger.WriteLine(); } } }
internal SummaryTable(Summary summary) { Summary = summary; if (summary.HasCriticalValidationErrors) { Columns = new SummaryTableColumn[0]; ColumnCount = 0; FullHeader = new string[0]; FullContent = new string[0][]; FullContentStartOfGroup = new bool[0]; FullContentWithHeader = new string[0][]; IsDefault = new bool[0]; return; } var columns = summary.GetColumns(); ColumnCount = columns.Length; FullHeader = columns.Select(c => c.ColumnName).ToArray(); var orderProvider = summary.Config.GetOrderProvider() ?? DefaultOrderProvider.Instance; FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.Benchmark)).ToArray()).ToArray(); IsDefault = columns.Select(c => summary.Reports.All(r => c.IsDefault(summary, r.Benchmark))).ToArray(); var groupKeys = summary.Benchmarks.Select(b => orderProvider.GetGroupKey(b, summary)).ToArray(); FullContentStartOfGroup = new bool[summary.Reports.Length]; if (groupKeys.Distinct().Count() > 1 && FullContentStartOfGroup.Length > 0) { FullContentStartOfGroup[0] = true; for (int i = 1; i < summary.Reports.Length; i++) FullContentStartOfGroup[i] = groupKeys[i] != groupKeys[i - 1]; } var full = new List<string[]> { FullHeader }; full.AddRange(FullContent); FullContentWithHeader = full.ToArray(); Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i].AlwaysShow)).ToArray(); }
public override void ExportToLog(Summary summary, ILogger logger) { var columns = lazyColumns.Value; logger.WriteLine(string.Join(Separator, columns.Select(c => CsvHelper.Escape(c.Title)))); foreach (var report in summary.Reports) { foreach (var measurement in report.AllMeasurements) { for (int i = 0; i < columns.Length; ) { logger.Write(CsvHelper.Escape(columns[i].GetValue(summary, report, measurement))); if (++i < columns.Length) { logger.Write(Separator); } } logger.WriteLine(); } } }
public string GetValue(Summary summary, Benchmark benchmark) { var baseline = summary.Benchmarks. Where(b => b.Job.DisplayInfo == benchmark.Job.DisplayInfo). Where(b => b.Parameters.DisplayInfo == benchmark.Parameters.DisplayInfo). FirstOrDefault(b => b.Target.Baseline); bool invalidResults = baseline == null || summary[baseline] == null || summary[baseline].ResultStatistics == null || !summary[baseline].ResultStatistics.CanBeInverted() || summary[benchmark] == null || summary[benchmark].ResultStatistics == null; if (invalidResults) return "?"; var baselineStat = summary[baseline].ResultStatistics; var targetStat = summary[benchmark].ResultStatistics; double mean = benchmark.Target.Baseline ? 1 : Statistics.DivMean(targetStat, baselineStat); double stdDev = benchmark.Target.Baseline ? 0 : Math.Sqrt(Statistics.DivVariance(targetStat, baselineStat)); switch (Kind) { case DiffKind.Mean: return mean.ToStr("N2"); case DiffKind.StdDev: return stdDev.ToStr("N2"); case DiffKind.WelchTTestPValue: { double pvalue = WelchTTest.Calc(baselineStat, targetStat).PValue; return pvalue > 0.0001 ? pvalue.ToStr("N4") : pvalue.ToStr("e2"); } default: throw new NotSupportedException(); } }
internal SummaryTable(Summary summary, SummaryStyle style = null) { Summary = summary; if (summary.HasCriticalValidationErrors) { Columns = Array.Empty <SummaryTableColumn>(); ColumnCount = 0; FullHeader = Array.Empty <string>(); FullContent = Array.Empty <string[]>(); FullContentStartOfHighlightGroup = Array.Empty <bool>(); FullContentWithHeader = Array.Empty <string[]>(); IsDefault = Array.Empty <bool>(); return; } // Ensure we have all required data for styling style = style ?? SummaryStyle.Default; if (style.TimeUnit == null) { style = style.WithTimeUnit(TimeUnit.GetBestTimeUnit(summary.Reports.Where(r => r.ResultStatistics != null).Select(r => r.ResultStatistics.Mean) .ToArray())); } if (style.SizeUnit == null) { style = style.WithSizeUnit(SizeUnit.GetBestSizeUnit(summary.Reports.Select(r => r.GcStats.BytesAllocatedPerOperation).ToArray())); } var columns = summary.GetColumns(); ColumnCount = columns.Length; FullHeader = columns.Select(c => c.GetColumnTitle(style)).ToArray(); FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.BenchmarkCase, style)).ToArray()).ToArray(); IsDefault = columns.Select(c => summary.Reports.All(r => c.IsDefault(summary, r.BenchmarkCase))).ToArray(); var highlightGroupKeys = summary.BenchmarksCases.Select(b => b.Config.Orderer.GetHighlightGroupKey(b)).ToArray(); FullContentStartOfHighlightGroup = new bool[summary.Reports.Length]; if (highlightGroupKeys.Distinct().Count() > 1 && FullContentStartOfHighlightGroup.Length > 0) { FullContentStartOfHighlightGroup[0] = true; for (int i = 1; i < summary.Reports.Length; i++) { FullContentStartOfHighlightGroup[i] = highlightGroupKeys[i] != highlightGroupKeys[i - 1]; } } var logicalGroupKeys = summary.BenchmarksCases .Select(b => b.Config.Orderer.GetLogicalGroupKey(summary.BenchmarksCases, b)) .ToArray(); FullContentStartOfLogicalGroup = new bool[summary.Reports.Length]; if (logicalGroupKeys.Distinct().Count() > 1 && FullContentStartOfLogicalGroup.Length > 0) { FullContentStartOfLogicalGroup[0] = true; for (int i = 1; i < summary.Reports.Length; i++) { FullContentStartOfLogicalGroup[i] = logicalGroupKeys[i] != logicalGroupKeys[i - 1]; } } SeparateLogicalGroups = summary.Orderer.SeparateLogicalGroups; var full = new List <string[]> { FullHeader }; full.AddRange(FullContent); FullContentWithHeader = full.ToArray(); Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i])).ToArray(); EffectiveSummaryStyle = style; }