Esempio n. 1
0
        private static void DisplayDiff(IEnumerable <JobResults> allResults, IEnumerable <string> allNames)
        {
            // Use the first job results as the reference for metadata:
            var firstJob = allResults.FirstOrDefault();

            if (firstJob == null || firstJob.Jobs.Count < 1)
            {
                return;
            }

            foreach (var jobEntry in firstJob.Jobs)
            {
                var jobName   = jobEntry.Key;
                var jobResult = jobEntry.Value;

                Console.WriteLine();

                // description + baseline job (firstJob) + other jobs * 2 (value + percentage)
                var table = new ResultTable(1 + 1 + (allNames.Count() - 1) * 2);

                table.Headers.Add(jobName);

                table.Headers.Add(allNames.First());

                foreach (var name in allNames.Skip(1))
                {
                    table.Headers.Add(name);
                    table.Headers.Add(""); // percentage
                }

                foreach (var metadata in jobResult.Metadata)
                {
                    var metadataKey = metadata.Name.Split(';').First();

                    if (!jobResult.Results.ContainsKey(metadataKey))
                    {
                        continue;
                    }

                    // We don't render the result if it's a raw object

                    if (metadata.Format == "object")
                    {
                        continue;
                    }

                    var row = table.AddRow();

                    var cell = new Cell();

                    cell.Elements.Add(new CellElement()
                    {
                        Text = metadata.Description, Alignment = CellTextAlignment.Left
                    });

                    row.Add(cell);

                    foreach (var result in allResults)
                    {
                        // Skip jobs that have no data for this measure
                        if (!result.Jobs.ContainsKey(jobName))
                        {
                            foreach (var n in allNames)
                            {
                                row.Add(new Cell());
                            }

                            row.Add(new Cell());

                            continue;
                        }

                        var job = result.Jobs[jobName];

                        if (!String.IsNullOrEmpty(metadata.Format))
                        {
                            var measure  = Convert.ToDouble(job.Results.ContainsKey(metadataKey) ? job.Results[metadataKey] : 0);
                            var previous = Convert.ToDouble(firstJob.Jobs[jobName].Results.ContainsKey(metadataKey) ? jobResult.Results[metadataKey] : 0);

                            var improvement = measure == 0
                            ? 0
                            : (measure - previous) / previous * 100;

                            row.Add(cell = new Cell());

                            cell.Elements.Add(new CellElement {
                                Text = Convert.ToDouble(measure).ToString(metadata.Format), Alignment = CellTextAlignment.Right
                            });

                            // Don't render % on baseline job
                            if (firstJob != result)
                            {
                                row.Add(cell = new Cell());

                                if (measure != 0)
                                {
                                    var sign = improvement > 0 ? "+" : "";
                                    cell.Elements.Add(new CellElement {
                                        Text = $"{sign}{improvement:n2}%", Alignment = CellTextAlignment.Right
                                    });
                                }
                            }
                        }
                        else
                        {
                            var measure = job.Results.ContainsKey(metadataKey) ? job.Results[metadataKey] : 0;

                            row.Add(cell = new Cell());
                            cell.Elements.Add(new CellElement {
                                Text = measure.ToString(), Alignment = CellTextAlignment.Right
                            });

                            // Don't render % on baseline job
                            if (firstJob != result)
                            {
                                row.Add(new Cell());
                            }
                        }
                    }
                }

                table.Render(Console.Out);

                Console.WriteLine();
            }
        }
Esempio n. 2
0
        private static void DisplayDiff(IEnumerable <Benchmark[]> allBenchmarks, IEnumerable <string> allNames)
        {
            // Use the first job's benchmarks as the reference for the rows:
            var firstBenchmarks = allBenchmarks.FirstOrDefault();

            if (firstBenchmarks == null || firstBenchmarks.Length < 1)
            {
                return;
            }

            var summaries = new Dictionary <string, List <BenchmarkSummary> >();

            foreach (var benchmark in firstBenchmarks)
            {
                summaries[benchmark.FullName] = new List <BenchmarkSummary>();
            }

            foreach (var benchmarks in allBenchmarks)
            {
                foreach (var benchmark in benchmarks)
                {
                    summaries[benchmark.FullName].Add(new BenchmarkSummary()
                    {
                        Name                         = benchmark.FullName,
                        MeanNanoseconds              = benchmark.Statistics.Mean,
                        StandardErrorNanoseconds     = benchmark.Statistics.StandardError,
                        StandardDeviationNanoseconds = benchmark.Statistics.StandardDeviation,
                        MedianNanoseconds            = benchmark.Statistics.Median,
                        Gen0                         = benchmark.Memory?.Gen0Collections ?? 0,
                        Gen1                         = benchmark.Memory?.Gen1Collections ?? 0,
                        Gen2                         = benchmark.Memory?.Gen2Collections ?? 0,
                        AllocatedBytes               = benchmark.Memory?.BytesAllocatedPerOperation ?? 0
                    });
                }
            }

            // Simplfy the benchmarks' names where possible to remove prefixes that
            // are all the same to reduce the width of the first column of the table
            // to the shortest unique string required across all benchmarks.
            var nameSegments = summaries.Keys.ToDictionary(key => key, value => value.Split('.'));

            while (true)
            {
                var areAllFirstSegmentsTheSame = nameSegments.Values
                                                 .Select(segments => segments[0])
                                                 .Distinct()
                                                 .Count() == 1;

                if (!areAllFirstSegmentsTheSame)
                {
                    // The names cannot be simplified further
                    break;
                }

                foreach (var pair in nameSegments)
                {
                    nameSegments[pair.Key] = pair.Value.Skip(1).ToArray();
                }
            }

            // Map the full names to their simplified name
            var simplifiedNames = nameSegments.ToDictionary(key => key.Key, value => string.Join(".", value.Value));

            var anyAllocations = summaries.Values
                                 .SelectMany(list => list)
                                 .Select(summary => summary.AllocatedBytes)
                                 .Any(allocatedBytes => allocatedBytes > 0);

            // Name + baseline mean (firstBenchmarks) + (other benchmarks' mean * 2 (value + ratio)) +
            // baseline allocations + (other benchmarks' mean * 2 (value + ratio))
            var otherCount = allNames.Count() - 1;
            var table      = new ResultTable(1 + 1 + (anyAllocations ? 1 : 0) + (otherCount * (anyAllocations ? 4 : 2)));

            var firstName = allNames.First();

            table.Headers.Add("benchmark");
            table.Headers.Add($"mean ({firstName})");

            foreach (var name in allNames.Skip(1))
            {
                table.Headers.Add($"mean ({name})");
                table.Headers.Add("ratio");
            }

            if (anyAllocations)
            {
                table.Headers.Add($"allocated ({firstName})");

                foreach (var name in allNames.Skip(1))
                {
                    table.Headers.Add($"allocated ({name})");
                    table.Headers.Add("ratio");
                }
            }

            foreach (var benchmark in summaries.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value))
            {
                var firstBenchmark = benchmark.First();
                var simplifiedName = simplifiedNames[firstBenchmark.Name];

                var benchmarks = summaries[firstBenchmark.Name];

                var row  = table.AddRow();
                var cell = new Cell();
                cell.Elements.Add(new CellElement()
                {
                    Text = simplifiedName, Alignment = CellTextAlignment.Left
                });
                row.Add(cell);

                AddCells(row, summary => summary.MeanNanoseconds, UnitType.Time, benchmarks);

                if (anyAllocations)
                {
                    AddCells(row, summary => summary.AllocatedBytes, UnitType.Size, benchmarks);
                }
            }

            Console.WriteLine("```md"); // Format as a GitHub-flavored Markdown table

            table.Render(Console.Out);

            Console.WriteLine("```");

            void AddCells(
                List <Cell> row,
                Func <BenchmarkSummary, object> valueFactory,
                UnitType unitType,
                List <BenchmarkSummary> summaries)
            {
                var rawValues = summaries
                                .Select(summary => valueFactory(summary))
                                .Select(value => Convert.ToDouble(value))
                                .ToArray();

                var precision = PrecisionHelper.GetPrecision(rawValues);
                var sizeUnit  = unitType == UnitType.Size ? SizeUnit.GetBestSizeUnit(rawValues) : null;
                var timeUnit  = unitType == UnitType.Time ? TimeUnit.GetBestTimeUnit(rawValues) : null;

                var units = unitType switch
                {
                    UnitType.Size => sizeUnit.Name,
                    UnitType.Time => timeUnit.Name,
                    _ => string.Empty
                };

                var baseline = summaries[0];

                for (var i = 0; i < summaries.Count; i++)
                {
                    var measure  = rawValues[i];
                    var previous = rawValues[0];

                    var ratio = measure == 0
                    ? 0
                    : measure / previous;

                    var formattedValue = unitType switch
                    {
                        UnitType.Size => new SizeValue((long)measure).ToString(sizeUnit),
                        UnitType.Time => new TimeInterval(measure).ToString(timeUnit, precision),
                        _ => measure.ToString("N" + precision, CultureInfo.InvariantCulture)
                    };

                    var cell = new Cell();
                    cell.Elements.Add(new CellElement
                    {
                        Text      = $"{formattedValue} {units}",
                        Alignment = CellTextAlignment.Right
                    });
                    row.Add(cell);

                    // Don't render the ratio on baseline benchmark
                    if (summaries[i] != baseline)
                    {
                        row.Add(cell = new Cell());

                        if (measure != 0)
                        {
                            cell.Elements.Add(new CellElement
                            {
                                Text      = ratio.ToString("N2", CultureInfo.InvariantCulture),
                                Alignment = CellTextAlignment.Right
                            });
                        }
                    }
                }
            }
        }