Example #1
0
        private List <BenchmarkReport> Run(List <Benchmark> benchmarks, IBenchmarkLogger logger)
        {
            logger.WriteLineHeader("// ***** BenchmarkRunner: Start   *****");
            logger.WriteLineInfo("// Found benchmarks:");
            foreach (var benchmark in benchmarks)
            {
                logger.WriteLineInfo($"//   {benchmark.Description}");
            }
            logger.NewLine();

            var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();

            var reports = new List <BenchmarkReport>();

            foreach (var benchmark in benchmarks)
            {
                if (benchmark.Task.ParametersSets.IsEmpty())
                {
                    var report = Run(logger, benchmark, importantPropertyNames);
                    reports.Add(report);
                    if (report.Runs.Count > 0)
                    {
                        var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
                        logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
                        logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
                    }
                }
                else
                {
                    var parametersSets = benchmark.Task.ParametersSets;
                    foreach (var parameters in parametersSets.ToParameters())
                    {
                        var report = Run(logger, benchmark, importantPropertyNames, parameters);
                        reports.Add(report);
                        if (report.Runs.Count > 0)
                        {
                            var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
                            logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
                            logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
                        }
                    }
                }
                logger.NewLine();
            }
            logger.WriteLineHeader("// ***** BenchmarkRunner: Finish  *****");
            logger.NewLine();

            BenchmarkMarkdownExporter.Default.Export(reports, logger);

            var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();

            if (warnings.Count > 0)
            {
                logger.NewLine();
                logger.WriteLineError("// *** Warnings *** ");
                foreach (var warning in warnings)
                {
                    logger.WriteLineError($"{warning.Message}");
                }
            }

            logger.NewLine();
            logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
            return(reports);
        }
        public IEnumerable <BenchmarkReport> RunCompetition(List <Benchmark> benchmarks)
        {
            benchmarks.Sort((a, b) => string.Compare((a.Task.Configuration.Caption + a.Target.Caption), b.Task.Configuration.Caption + b.Target.Caption, StringComparison.Ordinal));
            Logger.WriteLineHeader("// ***** Competition: Start   *****");
            Logger.WriteLineInfo("// Found benchmarks:");
            foreach (var benchmark in benchmarks)
            {
                Logger.WriteLineInfo($"//   {benchmark.Caption} {benchmark.Task.Settings.ToArgs()}");
            }
            Logger.NewLine();

            var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();

            var reports = new List <BenchmarkReport>();

            foreach (var benchmark in benchmarks)
            {
                var report = Run(benchmark, importantPropertyNames);
                reports.Add(report);
                if (report.Runs.Count > 0)
                {
                    var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
                    Logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
                    Logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
                }
                Logger.NewLine();
            }
            Logger.WriteLineHeader("// ***** Competition: Finish  *****");
            Logger.NewLine();
            Logger.WriteLineInfo("```ini");
            Logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
            var reportStats = reports.Where(r => r.Runs.Count > 0).Select(
                r => new
            {
                r.Benchmark,
                Stat = new BenchmarkRunReportsStatistic("Target", r.Runs)
            }).ToList();

            // Ensure uniform number formats and use of time units via these helpers.
            var averageTimeStats      = reportStats.Select(reportStat => reportStat.Stat);
            var timeToStringFunc      = GetTimeMeasurementFormattingFunc(averageTimeStats);
            var opsPerSecToStringFunc = GetOpsPerSecFormattingFunc();

            var table = new List <string[]> {
                new[] { "Type", "Method", "Mode", "Platform", "Jit", ".NET", "AvrTime", "StdDev", "op/s" }
            };

            foreach (var reportStat in reportStats)
            {
                var b = reportStat.Benchmark;

                string[] row =
                {
                    b.Target.Type.Name,
                    b.Target.Method.Name,
                    b.Task.Configuration.Mode.ToString(),
                    b.Task.Configuration.Platform.ToString(),
                    b.Task.Configuration.JitVersion.ToString(),
                    b.Task.Configuration.Framework.ToString(),
                    timeToStringFunc(reportStat.Stat.AverageTime.Median),
                    timeToStringFunc(reportStat.Stat.AverageTime.StandardDeviation),
                    opsPerSecToStringFunc(reportStat.Stat.OperationsPerSeconds.Median)
                };
                table.Add(row);
            }
            PrintTable(table);
            var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();

            if (benchmarksWithTroubles.Count > 0)
            {
                Logger.NewLine();
                Logger.WriteLineError("Benchmarks with troubles:");
                foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
                {
                    Logger.WriteLineError("  " + benchmarkWithTroubles.Caption);
                }
            }
            Logger.NewLine();
            Logger.WriteLineHeader("// ***** Competition: End *****");
            return(reports);
        }