public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, Func <Job, IToolchain> toolchainProvider) { var resolver = DefaultResolver; var benchmarks = benchmarkRunInfo.Benchmarks; var config = benchmarkRunInfo.Config; var title = GetTitle(benchmarks); var rootArtifactsFolderPath = (config?.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath).CreateIfNotExists(); using (var logStreamWriter = Portability.StreamWriter.FromPath(Path.Combine(rootArtifactsFolderPath, title + ".log"))) { var logger = new CompositeLogger(config.GetCompositeLogger(), new StreamLogger(logStreamWriter)); benchmarks = GetSupportedBenchmarks(benchmarks, logger, toolchainProvider, resolver); var artifactsToCleanup = new List <string>(); try { var runInfo = new BenchmarkRunInfo(benchmarks, benchmarkRunInfo.Type, config); return(Run(runInfo, logger, title, rootArtifactsFolderPath, toolchainProvider, resolver, artifactsToCleanup)); } finally { logger.WriteLineHeader("// * Artifacts cleanup *"); Cleanup(artifactsToCleanup); } } }
public static Summary Run(BenchmarkRunInfo[] benchmarkRunInfos, Func <Job, IToolchain> toolchainProvider) { var temp = benchmarkRunInfos.FirstOrDefault(); var benchmarkRunInfo = new BenchmarkRunInfo( benchmarkRunInfos.SelectMany(i => i.Benchmarks).ToArray(), temp?.Type, temp?.Config); return(Run(benchmarkRunInfo, toolchainProvider)); }
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, ILogger logger, string title, string rootArtifactsFolderPath, Func <Job, IToolchain> toolchainProvider, IResolver resolver, List <string> artifactsToCleanup) { var benchmarks = benchmarkRunInfo.Benchmarks; var config = benchmarkRunInfo.Config; logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); logger.WriteLineInfo("// Found benchmarks:"); foreach (var benchmark in benchmarks) { logger.WriteLineInfo($"// {benchmark.DisplayInfo}"); } logger.WriteLine(); var validationErrors = Validate(benchmarks, logger, config); if (validationErrors.Any(validationError => validationError.IsCritical)) { return(Summary.CreateFailed(benchmarks, title, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors)); } var globalChronometer = Chronometer.Start(); var reports = new List <BenchmarkReport>(); var buildResults = BuildInParallel(logger, rootArtifactsFolderPath, toolchainProvider, resolver, benchmarks, config, ref globalChronometer); foreach (var benchmark in benchmarks) { var buildResult = buildResults[benchmark]; if (!config.KeepBenchmarkFiles) { artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup); } if (buildResult.IsBuildSuccess) { var report = RunCore(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver, buildResult); reports.Add(report); if (report.GetResultRuns().Any()) { logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr()); } } else { reports.Add(new BenchmarkReport(benchmark, buildResult, buildResult, null, null, default)); if (buildResult.GenerateException != null) { logger.WriteLineError($"// Generate Exception: {buildResult.GenerateException.Message}"); } if (buildResult.BuildException != null) { logger.WriteLineError($"// Build Exception: {buildResult.BuildException.Message}"); } } logger.WriteLine(); } var clockSpan = globalChronometer.GetElapsed(); var summary = new Summary(title, reports, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors); logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****"); logger.WriteLine(); logger.WriteLineHeader("// * Export *"); var currentDirectory = Directory.GetCurrentDirectory(); foreach (var file in config.GetCompositeExporter().ExportToFiles(summary, logger)) { logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}"); } logger.WriteLine(); logger.WriteLineHeader("// * Detailed results *"); // TODO: make exporter foreach (var report in reports) { logger.WriteLineInfo(report.Benchmark.DisplayInfo); logger.WriteLineStatistic($"Runtime = {report.GetRuntimeInfo()}; GC = {report.GetGcInfo()}"); var resultRuns = report.GetResultRuns(); if (resultRuns.IsEmpty()) { logger.WriteLineError("There are not any results runs"); } else { logger.WriteLineStatistic(resultRuns.GetStatistics().ToTimeStr()); } logger.WriteLine(); } LogTotalTime(logger, clockSpan.GetTimeSpan()); logger.WriteLine(); logger.WriteLineHeader("// * Summary *"); MarkdownExporter.Console.ExportToLog(summary, logger); // TODO: make exporter ConclusionHelper.Print(logger, config.GetCompositeAnalyser().Analyse(summary).ToList()); // TODO: move to conclusions var columnWithLegends = summary.Table.Columns.Select(c => c.OriginalColumn).Where(c => !string.IsNullOrEmpty(c.Legend)).ToList(); var effectiveTimeUnit = summary.Table.EffectiveSummaryStyle.TimeUnit; if (columnWithLegends.Any() || effectiveTimeUnit != null) { logger.WriteLine(); logger.WriteLineHeader("// * Legends *"); int maxNameWidth = 0; if (columnWithLegends.Any()) { maxNameWidth = Math.Max(maxNameWidth, columnWithLegends.Select(c => c.ColumnName.Length).Max()); } if (effectiveTimeUnit != null) { maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.Name.Length + 2); } foreach (var column in columnWithLegends) { logger.WriteLineHint($" {column.ColumnName.PadRight(maxNameWidth, ' ')} : {column.Legend}"); } if (effectiveTimeUnit != null) { logger.WriteLineHint($" {("1 " + effectiveTimeUnit.Name).PadRight(maxNameWidth, ' ')} :" + $" 1 {effectiveTimeUnit.Description} ({TimeUnit.Convert(1, effectiveTimeUnit, TimeUnit.Second).ToStr("0.#########")} sec)"); } } if (config.GetDiagnosers().Any()) { logger.WriteLine(); config.GetCompositeDiagnoser().DisplayResults(logger); } logger.WriteLine(); logger.WriteLineHeader("// ***** BenchmarkRunner: End *****"); return(summary); }
private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, Dictionary <BenchmarkCase, (BenchmarkId benchmarkId, BuildResult buildResult)> buildResults,
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo) { using (DirtyAssemblyResolveHelper.Create()) return(RunWithExceptionHandling(() => RunWithDirtyAssemblyResolveHelper(new[] { benchmarkRunInfo }).Single())); }
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, bool summaryPerType) => Run(new[] { benchmarkRunInfo }, benchmarkRunInfo.Config, summaryPerType).Single();
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo) => Run(new[] { benchmarkRunInfo }, benchmarkRunInfo.Config).Single();
public static Summary Run(BenchmarkRunInfo benchmarks) => BenchmarkRunnerCore.Run(benchmarks, ToolchainExtensions.GetToolchain);