Ejemplo n.º 1
0
        public override void ExportToLog(Summary summary, ILogger logger)
        {
            logger.WriteLine($"<pre><code>{EnvironmentHelper.GetCurrentInfo().ToFormattedString("Host")}</code></pre>");
            logger.WriteLine();

            PrintTable(summary.Table, logger);
        }
Ejemplo n.º 2
0
        internal SummaryTable(Summary summary)
        {
            Summary = summary;

            var configColumns = summary.Config.
                GetColumns().
                Where(c => c.IsAvailable(summary));
            var paramColumns = summary.Benchmarks.
                SelectMany(b => b.Parameters.Items.Select(item => item.Name)).
                Distinct().
                Select(name => new ParamColumn(name));
            var columns = configColumns.Concat(paramColumns).ToArray();

            ColumnCount = columns.Length;
            FullHeader = columns.Select(c => c.ColumnName).ToArray();

            var reports = summary.Reports.Values.
                OrderBy(r => r.Benchmark.Parameters.FullInfo).
                ThenBy(r => r.Benchmark.Target.Type.Name).
                ToList();
            FullContent = reports.Select(r => columns.Select(c => c.GetValue(summary, r.Benchmark)).ToArray()).ToArray();

            var full = new List<string[]> { FullHeader };
            full.AddRange(FullContent);
            FullContentWithHeader = full.ToArray();

            Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i].AlwaysShow)).ToArray();
        }
 public override void ExportToLog(Summary summary, ILogger logger)
 {
     logger.WriteLine(string.Join(";", columns.Select(c => c.Title)));
     foreach (var report in summary.Reports.Values)
         foreach (var measurement in report.AllMeasurements)
             logger.WriteLine(string.Join(";", columns.Select(column => column.GetValue(summary, report, measurement))));
 }
 public IEnumerable<IColumn> GetColumns(Summary summary)
 {
     return summary.Benchmarks.
         SelectMany(b => b.Parameters.Items.Select(item => item.Name)).
         Distinct().
         Select(name => new ParamColumn(name));
 }
        public override void ExportToLog(Summary summary, ILogger logger)
        {
            logger.WriteLine("<!DOCTYPE html>");
            logger.WriteLine("<html lang='en'>");
            logger.WriteLine("<head>");
            logger.WriteLine("<meta charset='utf-8' />");
            logger.WriteLine("<title>" + summary.Title + "</title>");
            logger.WriteLine(CssDefinition);
            logger.WriteLine("</head>");

            logger.WriteLine("<body>");
            logger.Write("<pre><code>");
            logger.WriteLine();
            foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString())
            {
                logger.WriteLine(infoLine);
            }
            logger.WriteLine(summary.JobRuntimes);
            logger.Write("</code></pre>");
            logger.WriteLine();

            PrintTable(summary.Table, logger);

            logger.WriteLine("</body>");
            logger.WriteLine("</html>");
        }
        public override void ExportToLog(Summary summary, ILogger logger)
        {
            if (useCodeBlocks)
            {
                logger.WriteLine(codeBlockStart);
            }
            logger = GetRightLogger(logger);
            logger.WriteLine();
            foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString())
            {
                logger.WriteLineInfo(infoLine);
            }
            logger.WriteLineInfo(summary.JobRuntimes);
            logger.WriteLine();

            PrintTable(summary.Table, logger);

            // TODO: move this logic to an analyser
            var benchmarksWithTroubles = summary.Reports.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList();
            if (benchmarksWithTroubles.Count > 0)
            {
                logger.WriteLine();
                logger.WriteLineError("Benchmarks with issues:");
                foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
                {
                    logger.WriteLineError("  " + benchmarkWithTroubles.DisplayInfo);
                }
            }
        }
Ejemplo n.º 7
0
 public string GetValue(Summary summary, Benchmark benchmark)
 {
     var ranks = RankHelper.GetRanks(summary.Reports.Select(r => r.ResultStatistics).ToArray());
     int index = Array.IndexOf(summary.Reports.Select(r => r.Benchmark).ToArray(), benchmark);
     int rank = ranks[index];
     return system.ToPresentation(rank);
 }
Ejemplo n.º 8
0
 public IEnumerable<string> ExportToFiles(Summary summary)
 {
     var fileName = $"{Path.Combine(summary.CurrentDirectory, summary.Title)}-{FileCaption}{FileNameSuffix}.{FileExtension}";
     using (var stream = new StreamWriter(fileName))
         ExportToLog(summary, new StreamLogger(stream));
     yield return fileName;
 }
        public override void ExportToLog(Summary summary, ILogger logger)
        {
            logger.WriteLine("....");
            foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString())
            {
                logger.WriteLineInfo(infoLine);
            }
            logger.WriteLineInfo(summary.JobRuntimes);
            logger.WriteLine();

            PrintTable(summary.Table, logger);

            var benchmarksWithTroubles = summary.Reports
                .Where(r => !r.GetResultRuns().Any())
                .Select(r => r.Benchmark)
                .ToList();

            if (benchmarksWithTroubles.Count > 0)
            {
                logger.WriteLine();
                logger.WriteLine("[WARNING]");
                logger.WriteLineError(".Benchmarks with issues");
                logger.WriteLine("====");
                foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
                    logger.WriteLineError("* " + benchmarkWithTroubles.DisplayInfo);
                logger.WriteLine("====");
            }
        }
Ejemplo n.º 10
0
 public IEnumerable<string> ExportToFiles(Summary summary)
 {
     var fileNames = new List<string>();
     foreach (var exporter in exporters)
         fileNames.AddRange(exporter.ExportToFiles(summary));
     return fileNames;
 }
Ejemplo n.º 11
0
        public string GetValue(Summary summary, Benchmark benchmark)
        {
            var baselineBenchmark = summary.Benchmarks.
                Where(b => b.Job.GetFullInfo() == benchmark.Job.GetFullInfo()).
                Where(b => b.Parameters.FullInfo == benchmark.Parameters.FullInfo).
                FirstOrDefault(b => b.Target.Baseline);
            if (baselineBenchmark == null)
                return "?";

            var baselineMedian = summary.Reports[baselineBenchmark].ResultStatistics.Median;
            var currentMedian = summary.Reports[benchmark].ResultStatistics.Median;

            switch (Kind)
            {
                case DiffKind.Delta:
                    if (benchmark.Target.Baseline)
                        return "Baseline";
                    var diff = (currentMedian - baselineMedian)/baselineMedian*100.0;
                    return diff.ToStr("0.0") + "%";
                case DiffKind.Scaled:
                    var scale = currentMedian/baselineMedian;
                    return scale.ToStr("0.00");
                default:
                    return "?";
            }
        }
Ejemplo n.º 12
0
        public IEnumerable<string> ExportToFiles(Summary summary)
        {
            const string scriptFileName = "BuildPlots.R";
            yield return scriptFileName;

            var fileNamePrefix = Path.Combine(summary.ResultsDirectoryPath, summary.Title);
            var scriptFullPath = Path.Combine(summary.ResultsDirectoryPath, scriptFileName);
            var script = ResourceHelper.LoadTemplate(scriptFileName).Replace("$BenchmarkDotNetVersion$", BenchmarkDotNetInfo.FullTitle);
            lock (buildScriptLock)
                File.WriteAllText(scriptFullPath, script);

            var rHome = Environment.GetEnvironmentVariable("R_HOME") ?? @"C:\Program Files\R\R-3.2.3\bin\";
            if (Directory.Exists(rHome))
            {
                var start = new ProcessStartInfo
                {
                    UseShellExecute = false,
                    RedirectStandardOutput = false,
                    CreateNoWindow = true,
                    FileName = Path.Combine(rHome, "Rscript.exe"),
                    WorkingDirectory = summary.ResultsDirectoryPath,
                    Arguments = $"\"{scriptFullPath}\" \"{fileNamePrefix}-measurements.csv\""
                };
                using (var process = Process.Start(start))
                    process?.WaitForExit();
                yield return fileNamePrefix + "-boxplot.png";
                yield return fileNamePrefix + "-barplot.png";
            }
        }
Ejemplo n.º 13
0
 public IEnumerable<IWarning> Analyze(Summary summary)
 {
     var hostInfo = summary.HostEnvironmentHelper;
     if (hostInfo.HasAttachedDebugger)
         yield return new Warning("AttachedDebugger", "Benchmark was executed with attached debugger.", null);
     if (hostInfo.Configuration.Equals("DEBUG", StringComparison.InvariantCultureIgnoreCase))
         yield return new Warning("DebugConfiguration", "Benchmark was built in DEBUG configuration. Please, build it in RELEASE.", null);
 }
 public IEnumerable<IWarning> Analyse(Summary summary)
 {
     var hostInfo = summary.HostEnvironmentInfo;
     if (hostInfo.HasAttachedDebugger)
         yield return new Warning("AttachedDebugger", "Benchmark was executed with attached debugger.", null);
     if (hostInfo.Configuration.EqualsWithIgnoreCase("DEBUG"))
         yield return new Warning("DebugConfiguration", "Benchmark was built in DEBUG configuration. Please, build it in RELEASE.", null);
 }
 public IEnumerable<IColumn> GetColumns(Summary summary)
 {
     if (summary.Benchmarks.Select(b => b.Target.Type.Namespace).Distinct().Count() > 1)
         yield return TargetMethodColumn.Namespace;
     if (summary.Benchmarks.Select(b => b.Target.Type.Name).Distinct().Count() > 1)
         yield return TargetMethodColumn.Type;
     yield return TargetMethodColumn.Method;
 }
Ejemplo n.º 16
0
 public IEnumerable<Conclusion> Analyse(Summary summary)
 {
     foreach (var conclusion in AnalyseSummary(summary))
         yield return conclusion;
     foreach (var report in summary.Reports)
         foreach (var conclusion in AnalyseReport(report))
             yield return conclusion;
 }
Ejemplo n.º 17
0
        public IEnumerable<string> ExportToFiles(Summary summary)
        {
            var filePath = $"{Path.Combine(summary.ResultsDirectoryPath, summary.Title)}-{FileCaption}{FileNameSuffix}.{FileExtension}";
            using (var stream = Portability.StreamWriter.FromPath(filePath))
            {
                ExportToLog(summary, new StreamLogger(stream));
            }

            return new [] { filePath };
        }
Ejemplo n.º 18
0
        public override void ExportToLog(Summary summary, ILogger logger)
        {
            logger.Write("<pre><code>");
            logger.WriteLine();
            foreach (var infoLine in HostEnvironmentInfo.GetCurrent().ToFormattedString())
            {
                logger.WriteLine(infoLine);
            }
            logger.Write("</code></pre>");
            logger.WriteLine();

            PrintTable(summary.Table, logger);
        }
Ejemplo n.º 19
0
        internal SummaryTable(Summary summary)
        {
            Summary = summary;

            if (summary.HasCriticalValidationErrors)
            {
                Columns = new SummaryTableColumn[0];
                ColumnCount = 0;
                FullHeader = new string[0];
                FullContent = new string[0][];
                FullContentStartOfGroup = new bool[0];
                FullContentWithHeader = new string[0][];
                return;
            }

            var configColumns = summary.Config.
                GetColumns().
                Where(c => c.IsAvailable(summary));
            var paramColumns = summary.Benchmarks.
                SelectMany(b => b.Parameters.Items.Select(item => item.Name)).
                Distinct().
                Select(name => new ParamColumn(name));
            var diagnoserColumns = summary.Config.
                GetDiagnosers().
                Where(d => d is IColumnProvider).
                Cast<IColumnProvider>().
                SelectMany(cp => cp.GetColumns);
            var columns = configColumns.Concat(paramColumns).Concat(diagnoserColumns).ToArray().OrderBy(c => c.Category).ToArray();

            ColumnCount = columns.Length;
            FullHeader = columns.Select(c => c.ColumnName).ToArray();

            var orderProvider = summary.Config.GetOrderProvider() ?? DefaultOrderProvider.Instance;
            FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.Benchmark)).ToArray()).ToArray();
            var groupKeys = summary.Benchmarks.Select(b => orderProvider.GetGroupKey(b, summary)).ToArray();
            FullContentStartOfGroup = new bool[summary.Reports.Length];

            if (groupKeys.Distinct().Count() > 1 && FullContentStartOfGroup.Length > 0)
            {
                FullContentStartOfGroup[0] = true;
                for (int i = 1; i < summary.Reports.Length; i++)
                    FullContentStartOfGroup[i] = groupKeys[i] != groupKeys[i - 1];
            }            

            var full = new List<string[]> { FullHeader };
            full.AddRange(FullContent);
            FullContentWithHeader = full.ToArray();

            Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i].AlwaysShow)).ToArray();
        }
        public IEnumerable<IWarning> Analyse(Summary summary)
        {
            if (summary.HostEnvironmentInfo.HasAttachedDebugger)
                yield return new Warning("AttachedDebugger", "Benchmark was executed with attached debugger.", null);

            foreach (var benchmark in summary.Benchmarks
                .Where(benchmark => benchmark.Target.Type.GetTypeInfo().Assembly.IsDebug().IsTrue()))
            {
                yield return
                    new Warning(
                        "DebugConfiguration",
                        "Benchmark was built in DEBUG configuration. Please, build it in RELEASE.",
                        summary.Reports.FirstOrDefault(report => ReferenceEquals(report.Benchmark, benchmark)));
            }
        }
        public string GetValue(Summary summary, Benchmark benchmark)
        {
            var baselineBenchmark = summary.Benchmarks.
                Where(b => b.Job.GetFullInfo() == benchmark.Job.GetFullInfo()).
                Where(b => b.Parameters.FullInfo == benchmark.Parameters.FullInfo).
                FirstOrDefault(b => b.Target.Baseline);
            var invalidResults = baselineBenchmark == null ||
                                 summary[baselineBenchmark] == null ||
                                 summary[baselineBenchmark].ResultStatistics == null ||
                                 summary[benchmark] == null ||
                                 summary[benchmark].ResultStatistics == null;

            if (invalidResults)
                return "?";

            double baselineMetric;
            double currentMetric;

            var resultStatistics = summary[baselineBenchmark].ResultStatistics;
            var statistics = summary[benchmark].ResultStatistics;
            if (Percentile == null)
            {
                baselineMetric = resultStatistics.Median;
                currentMetric = statistics.Median;
            }
            else
            {
                baselineMetric = resultStatistics.Percentiles.Percentile(Percentile.GetValueOrDefault());
                currentMetric = statistics.Percentiles.Percentile(Percentile.GetValueOrDefault());
            }

            if (baselineMetric == 0)
                return "?";

            switch (Kind)
            {
                case DiffKind.Delta:
                    if (benchmark.Target.Baseline)
                        return "Baseline";
                    var diff = (currentMetric - baselineMetric) / baselineMetric * 100.0;
                    return diff.ToStr("N1") + "%";
                case DiffKind.Scaled:
                    var scale = currentMetric / baselineMetric;
                    return scale.ToStr("N2");
                default:
                    return "?";
            }
        }
Ejemplo n.º 22
0
 public string GetValue(Summary summary, Benchmark benchmark)
 {
     var sortedBenchmarks = summary.Benchmarks.
         OrderBy(b => summary.Reports[b].ResultStatistics.Mean).
         ToArray();
     var places = GetPlaces(sortedBenchmarks.Select(b => summary.Reports[b].ResultStatistics).ToArray());
     var place = places[Array.IndexOf(sortedBenchmarks, benchmark)];
     switch (kind)
     {
         case Kind.ArabicNumber:
             return place.ToString();
         case Kind.Stars:
             return new string('*', place);
     }
     return place.ToString();
 }
            public IEnumerable<IColumn> GetColumns(Summary summary)
            {
                yield return StatisticColumn.Mean;

                if (NeedToShow(summary, s => s.StandardError > s.Mean * 0.01)) // TODO: Use Accuracy.MaxStdErrRelative
                    yield return StatisticColumn.StdErr;
                if (NeedToShow(summary, s => s.Percentiles.P95 > s.Mean + 3 * s.StandardDeviation))
                    yield return StatisticColumn.P95;
                if (NeedToShow(summary, s=> s.GetConfidenceInterval(ConfidenceLevel.L99).Contains(s.Median)))
                    yield return StatisticColumn.Median;
                if (NeedToShow(summary, s => s.StandardDeviation > 1e-9))
                    yield return StatisticColumn.StdDev;

                yield return BaselineScaledColumn.Scaled;
                yield return BaselineScaledColumn.ScaledStdDev; // TODO: make optional
            }
Ejemplo n.º 24
0
        public override void ExportToLog(Summary summary, ILogger logger)
        {
            foreach (var line in summary.Table.FullContentWithHeader)
            {
                for (int i = 0; i < line.Length;)
                {
                    logger.Write(CsvHelper.Escape(line[i]));

                    if (++i < line.Length)
                    {
                        logger.Write(separator);
                    }
                }

                logger.WriteLine();
            }
        }
Ejemplo n.º 25
0
 public override void ExportToLog(Summary summary, ILogger logger)
 {
     foreach (var report in summary.Reports.Values)
     {
         var runs = report.AllMeasurements;
         var modes = runs.Select(it => it.IterationMode).Distinct();
         logger.WriteLineHeader($"*** {report.Benchmark.ShortInfo} ***");
         logger.WriteLineHeader("* Raw *");
         foreach (var run in runs)
             logger.WriteLineResult(run.ToStr());
         foreach (var mode in modes)
         {
             logger.NewLine();
             logger.WriteLineHeader($"* Statistics for {mode}");
             logger.WriteLineStatistic(runs.Where(it => it.IterationMode == mode).GetStatistics().ToTimeStr());
         }
     }
 }
Ejemplo n.º 26
0
        public override void ExportToLog(Summary summary, ILogger logger)
        {
            if (useCodeBlocks)
                logger.WriteLine($"```{codeBlocksSyntax}");
            logger = new LoggerWithPrefix(logger, prefix);
            logger.WriteLineInfo(EnvironmentHelper.GetCurrentInfo().ToFormattedString("Host"));
            logger.NewLine();

            PrintTable(summary.Table, logger);

            // TODO: move this logic to an analyser
            var benchmarksWithTroubles = summary.Reports.Values.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList();
            if (benchmarksWithTroubles.Count > 0)
            {
                logger.NewLine();
                logger.WriteLineError("Benchmarks with troubles:");
                foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
                    logger.WriteLineError("  " + benchmarkWithTroubles.ShortInfo);
            }
        }
        public override void ExportToLog(Summary summary, ILogger logger)
        {
            logger.WriteLine(string.Join(";", columns.Select(c => c.Title)));

            foreach (var report in summary.Reports)
            {
                foreach (var measurement in report.AllMeasurements)
                {
                    for (int i = 0; i < columns.Length; )
                    {
                        logger.Write(columns[i].GetValue(summary, report, measurement));

                        if (++i < columns.Length)
                        {
                            logger.Write(";");
                        }
                    }
                    logger.WriteLine();
                }
            }
        }
Ejemplo n.º 28
0
        internal SummaryTable(Summary summary)
        {
            Summary = summary;

            if (summary.HasCriticalValidationErrors)
            {
                Columns = new SummaryTableColumn[0];
                ColumnCount = 0;
                FullHeader = new string[0];
                FullContent = new string[0][];
                FullContentStartOfGroup = new bool[0];
                FullContentWithHeader = new string[0][];
                IsDefault = new bool[0];
                return;
            }

            var columns = summary.GetColumns();

            ColumnCount = columns.Length;
            FullHeader = columns.Select(c => c.ColumnName).ToArray();

            var orderProvider = summary.Config.GetOrderProvider() ?? DefaultOrderProvider.Instance;
            FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.Benchmark)).ToArray()).ToArray();
            IsDefault = columns.Select(c => summary.Reports.All(r => c.IsDefault(summary, r.Benchmark))).ToArray();
            var groupKeys = summary.Benchmarks.Select(b => orderProvider.GetGroupKey(b, summary)).ToArray();
            FullContentStartOfGroup = new bool[summary.Reports.Length];

            if (groupKeys.Distinct().Count() > 1 && FullContentStartOfGroup.Length > 0)
            {
                FullContentStartOfGroup[0] = true;
                for (int i = 1; i < summary.Reports.Length; i++)
                    FullContentStartOfGroup[i] = groupKeys[i] != groupKeys[i - 1];
            }            

            var full = new List<string[]> { FullHeader };
            full.AddRange(FullContent);
            FullContentWithHeader = full.ToArray();

            Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i].AlwaysShow)).ToArray();
        }
        public override void ExportToLog(Summary summary, ILogger logger)
        {
            var columns = lazyColumns.Value;
            logger.WriteLine(string.Join(Separator, columns.Select(c => CsvHelper.Escape(c.Title))));

            foreach (var report in summary.Reports)
            {
                foreach (var measurement in report.AllMeasurements)
                {
                    for (int i = 0; i < columns.Length; )
                    {
                        logger.Write(CsvHelper.Escape(columns[i].GetValue(summary, report, measurement)));

                        if (++i < columns.Length)
                        {
                            logger.Write(Separator);
                        }
                    }
                    logger.WriteLine();
                }
            }
        }
        public string GetValue(Summary summary, Benchmark benchmark)
        {
            var baseline = summary.Benchmarks.
                Where(b => b.Job.DisplayInfo == benchmark.Job.DisplayInfo).
                Where(b => b.Parameters.DisplayInfo == benchmark.Parameters.DisplayInfo).
                FirstOrDefault(b => b.Target.Baseline);
            bool invalidResults = baseline == null ||
                                 summary[baseline] == null ||
                                 summary[baseline].ResultStatistics == null ||
                                 !summary[baseline].ResultStatistics.CanBeInverted() ||
                                 summary[benchmark] == null ||
                                 summary[benchmark].ResultStatistics == null;

            if (invalidResults)
                return "?";

            var baselineStat = summary[baseline].ResultStatistics;
            var targetStat = summary[benchmark].ResultStatistics;

            double mean = benchmark.Target.Baseline ? 1 : Statistics.DivMean(targetStat, baselineStat);
            double stdDev = benchmark.Target.Baseline ? 0 : Math.Sqrt(Statistics.DivVariance(targetStat, baselineStat));

            switch (Kind)
            {
                case DiffKind.Mean:
                    return mean.ToStr("N2");
                case DiffKind.StdDev:
                    return stdDev.ToStr("N2");
                case DiffKind.WelchTTestPValue:
                {
                    double pvalue = WelchTTest.Calc(baselineStat, targetStat).PValue;
                    return pvalue > 0.0001 ? pvalue.ToStr("N4") : pvalue.ToStr("e2");
                }
                default:
                    throw new NotSupportedException();
            }
        }
Ejemplo n.º 31
0
        internal SummaryTable(Summary summary, SummaryStyle style = null)
        {
            Summary = summary;

            if (summary.HasCriticalValidationErrors)
            {
                Columns     = Array.Empty <SummaryTableColumn>();
                ColumnCount = 0;
                FullHeader  = Array.Empty <string>();
                FullContent = Array.Empty <string[]>();
                FullContentStartOfHighlightGroup = Array.Empty <bool>();
                FullContentWithHeader            = Array.Empty <string[]>();
                IsDefault = Array.Empty <bool>();
                return;
            }

            // Ensure we have all required data for styling
            style = style ?? SummaryStyle.Default;
            if (style.TimeUnit == null)
            {
                style = style.WithTimeUnit(TimeUnit.GetBestTimeUnit(summary.Reports.Where(r => r.ResultStatistics != null).Select(r => r.ResultStatistics.Mean)
                                                                    .ToArray()));
            }

            if (style.SizeUnit == null)
            {
                style = style.WithSizeUnit(SizeUnit.GetBestSizeUnit(summary.Reports.Select(r => r.GcStats.BytesAllocatedPerOperation).ToArray()));
            }

            var columns = summary.GetColumns();

            ColumnCount = columns.Length;
            FullHeader  = columns.Select(c => c.GetColumnTitle(style)).ToArray();

            FullContent = summary.Reports.Select(r => columns.Select(c => c.GetValue(summary, r.BenchmarkCase, style)).ToArray()).ToArray();
            IsDefault   = columns.Select(c => summary.Reports.All(r => c.IsDefault(summary, r.BenchmarkCase))).ToArray();

            var highlightGroupKeys = summary.BenchmarksCases.Select(b => b.Config.Orderer.GetHighlightGroupKey(b)).ToArray();

            FullContentStartOfHighlightGroup = new bool[summary.Reports.Length];
            if (highlightGroupKeys.Distinct().Count() > 1 && FullContentStartOfHighlightGroup.Length > 0)
            {
                FullContentStartOfHighlightGroup[0] = true;
                for (int i = 1; i < summary.Reports.Length; i++)
                {
                    FullContentStartOfHighlightGroup[i] = highlightGroupKeys[i] != highlightGroupKeys[i - 1];
                }
            }

            var logicalGroupKeys = summary.BenchmarksCases
                                   .Select(b => b.Config.Orderer.GetLogicalGroupKey(summary.BenchmarksCases, b))
                                   .ToArray();

            FullContentStartOfLogicalGroup = new bool[summary.Reports.Length];
            if (logicalGroupKeys.Distinct().Count() > 1 && FullContentStartOfLogicalGroup.Length > 0)
            {
                FullContentStartOfLogicalGroup[0] = true;
                for (int i = 1; i < summary.Reports.Length; i++)
                {
                    FullContentStartOfLogicalGroup[i] = logicalGroupKeys[i] != logicalGroupKeys[i - 1];
                }
            }

            SeparateLogicalGroups = summary.Orderer.SeparateLogicalGroups;

            var full = new List <string[]> {
                FullHeader
            };

            full.AddRange(FullContent);
            FullContentWithHeader = full.ToArray();

            Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i])).ToArray();
            EffectiveSummaryStyle = style;
        }