public static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos, IConfig commonSettingsConfig, bool summaryPerType) { var resolver = DefaultResolver; var artifactsToCleanup = new List <string>(); var title = GetTitle(benchmarkRunInfos); var rootArtifactsFolderPath = (commonSettingsConfig?.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath).CreateIfNotExists(); using (var logStreamWriter = Portability.StreamWriter.FromPath(Path.Combine(rootArtifactsFolderPath, title + ".log"))) { var logger = new CompositeLogger(commonSettingsConfig.GetCompositeLogger(), new StreamLogger(logStreamWriter)); var supportedBenchmarks = GetSupportedBenchmarks(benchmarkRunInfos, logger, resolver); var validationErrors = Validate(supportedBenchmarks, logger); if (validationErrors.Any(validationError => validationError.IsCritical)) { return new [] { Summary.CreateFailed( supportedBenchmarks.SelectMany(b => b.Benchmarks).ToArray(), title, HostEnvironmentInfo.GetCurrent(), commonSettingsConfig, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors) } } ; var buildPartitions = BenchmarkPartitioner.CreateForBuild(supportedBenchmarks, resolver); logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); var globalChronometer = Chronometer.Start(); var buildResults = BuildInParallel(logger, rootArtifactsFolderPath, buildPartitions, ref globalChronometer); try { var results = new List <Summary>(); var benchmarkToBuildResult = buildResults .SelectMany(buildResult => buildResult.Key.Benchmarks.Select(buildInfo => (buildInfo.Benchmark, buildInfo.Id, buildResult.Value))) .ToDictionary(info => info.Benchmark, info => (info.Id, info.Value)); foreach (var benchmarkRunInfo in supportedBenchmarks) // we run them in the old order now using the new build artifacts { results.Add(Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, logger, artifactsToCleanup, rootArtifactsFolderPath, ref globalChronometer)); } return(results.ToArray()); } finally { logger.WriteLineHeader("// * Artifacts cleanup *"); Cleanup(new HashSet <string>(artifactsToCleanup.Distinct())); } } }
private static Summary Run(Benchmark[] benchmarks, ILogger logger, string title, IConfig config, string rootArtifactsFolderPath) { logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); logger.WriteLineInfo("// Found benchmarks:"); foreach (var benchmark in benchmarks) { logger.WriteLineInfo($"// {benchmark.ShortInfo}"); } logger.WriteLine(); var validationErrors = Validate(benchmarks, logger, config); if (validationErrors.Any(validationError => validationError.IsCritical)) { return(Summary.CreateFailed(benchmarks, title, EnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors)); } var globalChronometer = Chronometer.Start(); var reports = new List <BenchmarkReport>(); foreach (var benchmark in benchmarks) { var report = Run(benchmark, logger, config, rootArtifactsFolderPath); reports.Add(report); if (report.GetResultRuns().Any()) { logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr()); } logger.WriteLine(); } var clockSpan = globalChronometer.Stop(); var summary = new Summary(title, reports, EnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors); logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****"); logger.WriteLine(); logger.WriteLineHeader("// * Export *"); var currentDirectory = Directory.GetCurrentDirectory(); foreach (var file in config.GetCompositeExporter().ExportToFiles(summary)) { logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}"); } logger.WriteLine(); logger.WriteLineHeader("// * Detailed results *"); // TODO: make exporter foreach (var report in reports) { logger.WriteLineInfo(report.Benchmark.ShortInfo); logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr()); logger.WriteLine(); } LogTotalTime(logger, clockSpan.GetTimeSpan()); logger.WriteLine(); logger.WriteLineHeader("// * Summary *"); MarkdownExporter.Console.ExportToLog(summary, logger); // TODO: make exporter var warnings = config.GetCompositeAnalyser().Analyse(summary).ToList(); if (warnings.Count > 0) { logger.WriteLine(); logger.WriteLineError("// * Warnings * "); foreach (var warning in warnings) { logger.WriteLineError($"{warning.Message}"); } } if (config.GetDiagnosers().Count() > 0) { logger.WriteLine(); config.GetCompositeDiagnoser().DisplayResults(logger); } logger.WriteLine(); logger.WriteLineHeader("// ***** BenchmarkRunner: End *****"); return(summary); }
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, ILogger logger, string title, string rootArtifactsFolderPath, Func <Job, IToolchain> toolchainProvider, IResolver resolver, List <string> artifactsToCleanup) { var benchmarks = benchmarkRunInfo.Benchmarks; var config = benchmarkRunInfo.Config; logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); logger.WriteLineInfo("// Found benchmarks:"); foreach (var benchmark in benchmarks) { logger.WriteLineInfo($"// {benchmark.DisplayInfo}"); } logger.WriteLine(); var validationErrors = Validate(benchmarks, logger, config); if (validationErrors.Any(validationError => validationError.IsCritical)) { return(Summary.CreateFailed(benchmarks, title, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors)); } var globalChronometer = Chronometer.Start(); var reports = new List <BenchmarkReport>(); var buildResults = BuildInParallel(logger, rootArtifactsFolderPath, toolchainProvider, resolver, benchmarks, config, ref globalChronometer); foreach (var benchmark in benchmarks) { var buildResult = buildResults[benchmark]; if (!config.KeepBenchmarkFiles) { artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup); } if (buildResult.IsBuildSuccess) { var report = RunCore(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver, buildResult); reports.Add(report); if (report.GetResultRuns().Any()) { logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr()); } } else { reports.Add(new BenchmarkReport(benchmark, buildResult, buildResult, null, null, default)); if (buildResult.GenerateException != null) { logger.WriteLineError($"// Generate Exception: {buildResult.GenerateException.Message}"); } if (buildResult.BuildException != null) { logger.WriteLineError($"// Build Exception: {buildResult.BuildException.Message}"); } } logger.WriteLine(); } var clockSpan = globalChronometer.GetElapsed(); var summary = new Summary(title, reports, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors); logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****"); logger.WriteLine(); logger.WriteLineHeader("// * Export *"); var currentDirectory = Directory.GetCurrentDirectory(); foreach (var file in config.GetCompositeExporter().ExportToFiles(summary, logger)) { logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}"); } logger.WriteLine(); logger.WriteLineHeader("// * Detailed results *"); // TODO: make exporter foreach (var report in reports) { logger.WriteLineInfo(report.Benchmark.DisplayInfo); logger.WriteLineStatistic($"Runtime = {report.GetRuntimeInfo()}; GC = {report.GetGcInfo()}"); var resultRuns = report.GetResultRuns(); if (resultRuns.IsEmpty()) { logger.WriteLineError("There are not any results runs"); } else { logger.WriteLineStatistic(resultRuns.GetStatistics().ToTimeStr()); } logger.WriteLine(); } LogTotalTime(logger, clockSpan.GetTimeSpan()); logger.WriteLine(); logger.WriteLineHeader("// * Summary *"); MarkdownExporter.Console.ExportToLog(summary, logger); // TODO: make exporter ConclusionHelper.Print(logger, config.GetCompositeAnalyser().Analyse(summary).ToList()); // TODO: move to conclusions var columnWithLegends = summary.Table.Columns.Select(c => c.OriginalColumn).Where(c => !string.IsNullOrEmpty(c.Legend)).ToList(); var effectiveTimeUnit = summary.Table.EffectiveSummaryStyle.TimeUnit; if (columnWithLegends.Any() || effectiveTimeUnit != null) { logger.WriteLine(); logger.WriteLineHeader("// * Legends *"); int maxNameWidth = 0; if (columnWithLegends.Any()) { maxNameWidth = Math.Max(maxNameWidth, columnWithLegends.Select(c => c.ColumnName.Length).Max()); } if (effectiveTimeUnit != null) { maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.Name.Length + 2); } foreach (var column in columnWithLegends) { logger.WriteLineHint($" {column.ColumnName.PadRight(maxNameWidth, ' ')} : {column.Legend}"); } if (effectiveTimeUnit != null) { logger.WriteLineHint($" {("1 " + effectiveTimeUnit.Name).PadRight(maxNameWidth, ' ')} :" + $" 1 {effectiveTimeUnit.Description} ({TimeUnit.Convert(1, effectiveTimeUnit, TimeUnit.Second).ToStr("0.#########")} sec)"); } } if (config.GetDiagnosers().Any()) { logger.WriteLine(); config.GetCompositeDiagnoser().DisplayResults(logger); } logger.WriteLine(); logger.WriteLineHeader("// ***** BenchmarkRunner: End *****"); return(summary); }
public static Summary Run(Benchmark[] benchmarks, ILogger logger, string title, IConfig config, string rootArtifactsFolderPath, Func <Job, IToolchain> toolchainProvider, IResolver resolver) { logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); logger.WriteLineInfo("// Found benchmarks:"); foreach (var benchmark in benchmarks) { logger.WriteLineInfo($"// {benchmark.DisplayInfo}"); } logger.WriteLine(); var validationErrors = Validate(benchmarks, logger, config); if (validationErrors.Any(validationError => validationError.IsCritical)) { return(Summary.CreateFailed(benchmarks, title, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors)); } var globalChronometer = Chronometer.Start(); var reports = new List <BenchmarkReport>(); foreach (var benchmark in benchmarks) { var report = Run(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver); reports.Add(report); if (report.GetResultRuns().Any()) { logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr()); } logger.WriteLine(); } var clockSpan = globalChronometer.Stop(); var summary = new Summary(title, reports, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors); logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****"); logger.WriteLine(); logger.WriteLineHeader("// * Export *"); var currentDirectory = Directory.GetCurrentDirectory(); foreach (var file in config.GetCompositeExporter().ExportToFiles(summary)) { logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}"); } logger.WriteLine(); logger.WriteLineHeader("// * Detailed results *"); // TODO: make exporter foreach (var report in reports) { logger.WriteLineInfo(report.Benchmark.DisplayInfo); var resultRuns = report.GetResultRuns(); if (resultRuns.IsEmpty()) { logger.WriteLineError("There are no any results runs"); } else { logger.WriteLineStatistic(resultRuns.GetStatistics().ToTimeStr()); } logger.WriteLine(); } LogTotalTime(logger, clockSpan.GetTimeSpan()); logger.WriteLine(); logger.WriteLineHeader("// * Summary *"); MarkdownExporter.Console.ExportToLog(summary, logger); // TODO: make exporter ConclusionHelper.Print(logger, config.GetCompositeAnalyser().Analyse(summary).ToList()); if (config.GetDiagnosers().Any()) { logger.WriteLine(); config.GetCompositeDiagnoser().DisplayResults(logger); } logger.WriteLine(); logger.WriteLineHeader("// ***** BenchmarkRunner: End *****"); return(summary); }
[PublicAPI] public static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos, [CanBeNull] IConfig commonSettingsConfig) { if (commonSettingsConfig == null) { commonSettingsConfig = DefaultConfig.Instance; } var resolver = DefaultResolver; var artifactsToCleanup = new List <string>(); string title = GetTitle(benchmarkRunInfos); string rootArtifactsFolderPath = (commonSettingsConfig?.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath).CreateIfNotExists(); using (var logStreamWriter = StreamWriter.FromPath(Path.Combine(rootArtifactsFolderPath, title + ".log"))) { var logger = new CompositeLogger(commonSettingsConfig.GetCompositeLogger(), new StreamLogger(logStreamWriter)); var supportedBenchmarks = GetSupportedBenchmarks(benchmarkRunInfos, logger, resolver); if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any())) { return new [] { Summary.CreateFailed( supportedBenchmarks.SelectMany(b => b.BenchmarksCases).ToArray(), title, HostEnvironmentInfo.GetCurrent(), commonSettingsConfig, GetResultsFolderPath(rootArtifactsFolderPath), Array.Empty <ValidationError>()) } } ; var validationErrors = Validate(supportedBenchmarks, logger); if (validationErrors.Any(validationError => validationError.IsCritical)) { return new [] { Summary.CreateFailed( supportedBenchmarks.SelectMany(b => b.BenchmarksCases).ToArray(), title, HostEnvironmentInfo.GetCurrent(), commonSettingsConfig, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors) } } ; var buildPartitions = BenchmarkPartitioner.CreateForBuild(supportedBenchmarks, resolver); logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); var globalChronometer = Chronometer.Start(); var buildResults = BuildInParallel(logger, rootArtifactsFolderPath, buildPartitions, ref globalChronometer); try { var results = new List <Summary>(); var benchmarkToBuildResult = buildResults .SelectMany(buildResult => buildResult.Key.Benchmarks.Select(buildInfo => (buildInfo.BenchmarkCase, buildInfo.Id, buildResult.Value))) .ToDictionary(info => info.BenchmarkCase, info => (info.Id, info.Value)); foreach (var benchmarkRunInfo in supportedBenchmarks) // we run them in the old order now using the new build artifacts { var runChronometer = Chronometer.Start(); var summary = Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, logger, artifactsToCleanup, rootArtifactsFolderPath, ref runChronometer); if (commonSettingsConfig == null || commonSettingsConfig.SummaryPerType) { PrintSummary(logger, benchmarkRunInfo.Config, summary); } LogTotalTime(logger, runChronometer.GetElapsed().GetTimeSpan(), summary.GetNumberOfExecutedBenchmarks(), message: "Run time"); logger.WriteLine(); results.Add(summary); } if (commonSettingsConfig != null && !commonSettingsConfig.SummaryPerType) { var joinedSummary = Summary.Join(results, commonSettingsConfig, globalChronometer.GetElapsed()); PrintSummary(logger, commonSettingsConfig, joinedSummary); results.Clear(); results.Add(joinedSummary); } return(results.ToArray()); } finally { logger.WriteLineHeader("// * Artifacts cleanup *"); Cleanup(new HashSet <string>(artifactsToCleanup.Distinct())); } } }