public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, Func <Job, IToolchain> toolchainProvider)
        {
            var resolver   = DefaultResolver;
            var benchmarks = benchmarkRunInfo.Benchmarks;
            var config     = benchmarkRunInfo.Config;

            var title = GetTitle(benchmarks);
            var rootArtifactsFolderPath = (config?.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath).CreateIfNotExists();

            using (var logStreamWriter = Portability.StreamWriter.FromPath(Path.Combine(rootArtifactsFolderPath, title + ".log")))
            {
                var logger = new CompositeLogger(config.GetCompositeLogger(), new StreamLogger(logStreamWriter));
                benchmarks = GetSupportedBenchmarks(benchmarks, logger, toolchainProvider, resolver);
                var artifactsToCleanup = new List <string>();

                try
                {
                    var runInfo = new BenchmarkRunInfo(benchmarks, benchmarkRunInfo.Type, config);
                    return(Run(runInfo, logger, title, rootArtifactsFolderPath, toolchainProvider, resolver, artifactsToCleanup));
                }
                finally
                {
                    logger.WriteLineHeader("// * Artifacts cleanup *");
                    Cleanup(artifactsToCleanup);
                }
            }
        }
        public static Summary Run(BenchmarkRunInfo[] benchmarkRunInfos, Func <Job, IToolchain> toolchainProvider)
        {
            var temp             = benchmarkRunInfos.FirstOrDefault();
            var benchmarkRunInfo = new BenchmarkRunInfo(
                benchmarkRunInfos.SelectMany(i => i.Benchmarks).ToArray(),
                temp?.Type,
                temp?.Config);

            return(Run(benchmarkRunInfo, toolchainProvider));
        }
        public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, ILogger logger, string title, string rootArtifactsFolderPath, Func <Job, IToolchain> toolchainProvider, IResolver resolver, List <string> artifactsToCleanup)
        {
            var benchmarks = benchmarkRunInfo.Benchmarks;
            var config     = benchmarkRunInfo.Config;

            logger.WriteLineHeader("// ***** BenchmarkRunner: Start   *****");
            logger.WriteLineInfo("// Found benchmarks:");
            foreach (var benchmark in benchmarks)
            {
                logger.WriteLineInfo($"//   {benchmark.DisplayInfo}");
            }
            logger.WriteLine();

            var validationErrors = Validate(benchmarks, logger, config);

            if (validationErrors.Any(validationError => validationError.IsCritical))
            {
                return(Summary.CreateFailed(benchmarks, title, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors));
            }

            var globalChronometer = Chronometer.Start();
            var reports           = new List <BenchmarkReport>();

            var buildResults = BuildInParallel(logger, rootArtifactsFolderPath, toolchainProvider, resolver, benchmarks, config, ref globalChronometer);

            foreach (var benchmark in benchmarks)
            {
                var buildResult = buildResults[benchmark];

                if (!config.KeepBenchmarkFiles)
                {
                    artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup);
                }

                if (buildResult.IsBuildSuccess)
                {
                    var report = RunCore(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver, buildResult);
                    reports.Add(report);
                    if (report.GetResultRuns().Any())
                    {
                        logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
                    }
                }
                else
                {
                    reports.Add(new BenchmarkReport(benchmark, buildResult, buildResult, null, null, default));

                    if (buildResult.GenerateException != null)
                    {
                        logger.WriteLineError($"// Generate Exception: {buildResult.GenerateException.Message}");
                    }
                    if (buildResult.BuildException != null)
                    {
                        logger.WriteLineError($"// Build Exception: {buildResult.BuildException.Message}");
                    }
                }

                logger.WriteLine();
            }
            var clockSpan = globalChronometer.GetElapsed();

            var summary = new Summary(title, reports, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors);

            logger.WriteLineHeader("// ***** BenchmarkRunner: Finish  *****");
            logger.WriteLine();

            logger.WriteLineHeader("// * Export *");
            var currentDirectory = Directory.GetCurrentDirectory();

            foreach (var file in config.GetCompositeExporter().ExportToFiles(summary, logger))
            {
                logger.WriteLineInfo($"  {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}");
            }
            logger.WriteLine();

            logger.WriteLineHeader("// * Detailed results *");

            // TODO: make exporter
            foreach (var report in reports)
            {
                logger.WriteLineInfo(report.Benchmark.DisplayInfo);
                logger.WriteLineStatistic($"Runtime = {report.GetRuntimeInfo()}; GC = {report.GetGcInfo()}");
                var resultRuns = report.GetResultRuns();
                if (resultRuns.IsEmpty())
                {
                    logger.WriteLineError("There are not any results runs");
                }
                else
                {
                    logger.WriteLineStatistic(resultRuns.GetStatistics().ToTimeStr());
                }
                logger.WriteLine();
            }

            LogTotalTime(logger, clockSpan.GetTimeSpan());
            logger.WriteLine();

            logger.WriteLineHeader("// * Summary *");
            MarkdownExporter.Console.ExportToLog(summary, logger);

            // TODO: make exporter
            ConclusionHelper.Print(logger, config.GetCompositeAnalyser().Analyse(summary).ToList());

            // TODO: move to conclusions
            var columnWithLegends = summary.Table.Columns.Select(c => c.OriginalColumn).Where(c => !string.IsNullOrEmpty(c.Legend)).ToList();
            var effectiveTimeUnit = summary.Table.EffectiveSummaryStyle.TimeUnit;

            if (columnWithLegends.Any() || effectiveTimeUnit != null)
            {
                logger.WriteLine();
                logger.WriteLineHeader("// * Legends *");
                int maxNameWidth = 0;
                if (columnWithLegends.Any())
                {
                    maxNameWidth = Math.Max(maxNameWidth, columnWithLegends.Select(c => c.ColumnName.Length).Max());
                }
                if (effectiveTimeUnit != null)
                {
                    maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.Name.Length + 2);
                }

                foreach (var column in columnWithLegends)
                {
                    logger.WriteLineHint($"  {column.ColumnName.PadRight(maxNameWidth, ' ')} : {column.Legend}");
                }

                if (effectiveTimeUnit != null)
                {
                    logger.WriteLineHint($"  {("1 " + effectiveTimeUnit.Name).PadRight(maxNameWidth, ' ')} :" +
                                         $" 1 {effectiveTimeUnit.Description} ({TimeUnit.Convert(1, effectiveTimeUnit, TimeUnit.Second).ToStr("0.#########")} sec)");
                }
            }

            if (config.GetDiagnosers().Any())
            {
                logger.WriteLine();
                config.GetCompositeDiagnoser().DisplayResults(logger);
            }

            logger.WriteLine();
            logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
            return(summary);
        }
Пример #4
0
 private static Summary Run(BenchmarkRunInfo benchmarkRunInfo,
                            Dictionary <BenchmarkCase, (BenchmarkId benchmarkId, BuildResult buildResult)> buildResults,
Пример #5
0
 public static Summary Run(BenchmarkRunInfo benchmarkRunInfo)
 {
     using (DirtyAssemblyResolveHelper.Create())
         return(RunWithExceptionHandling(() => RunWithDirtyAssemblyResolveHelper(new[] { benchmarkRunInfo }).Single()));
 }
Пример #6
0
 public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, bool summaryPerType)
 => Run(new[] { benchmarkRunInfo }, benchmarkRunInfo.Config, summaryPerType).Single();
Пример #7
0
 public static Summary Run(BenchmarkRunInfo benchmarkRunInfo) => Run(new[] { benchmarkRunInfo }, benchmarkRunInfo.Config).Single();
Пример #8
0
 public static Summary Run(BenchmarkRunInfo benchmarks) => BenchmarkRunnerCore.Run(benchmarks, ToolchainExtensions.GetToolchain);