public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, Func <Job, IToolchain> toolchainProvider)
{
var resolver = DefaultResolver;
var benchmarks = benchmarkRunInfo.Benchmarks;
var config = benchmarkRunInfo.Config;
var title = GetTitle(benchmarks);
var rootArtifactsFolderPath = (config?.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath).CreateIfNotExists();
using (var logStreamWriter = Portability.StreamWriter.FromPath(Path.Combine(rootArtifactsFolderPath, title + ".log")))
{
var logger = new CompositeLogger(config.GetCompositeLogger(), new StreamLogger(logStreamWriter));
benchmarks = GetSupportedBenchmarks(benchmarks, logger, toolchainProvider, resolver);
var artifactsToCleanup = new List <string>();
try
{
var runInfo = new BenchmarkRunInfo(benchmarks, benchmarkRunInfo.Type, config);
return(Run(runInfo, logger, title, rootArtifactsFolderPath, toolchainProvider, resolver, artifactsToCleanup));
}
finally
{
logger.WriteLineHeader("// * Artifacts cleanup *");
Cleanup(artifactsToCleanup);
}
}
}
public static Summary Run(BenchmarkRunInfo[] benchmarkRunInfos, Func <Job, IToolchain> toolchainProvider)
{
var temp = benchmarkRunInfos.FirstOrDefault();
var benchmarkRunInfo = new BenchmarkRunInfo(
benchmarkRunInfos.SelectMany(i => i.Benchmarks).ToArray(),
temp?.Type,
temp?.Config);
return(Run(benchmarkRunInfo, toolchainProvider));
}
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, ILogger logger, string title, string rootArtifactsFolderPath, Func <Job, IToolchain> toolchainProvider, IResolver resolver, List <string> artifactsToCleanup)
{
var benchmarks = benchmarkRunInfo.Benchmarks;
var config = benchmarkRunInfo.Config;
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
{
logger.WriteLineInfo($"// {benchmark.DisplayInfo}");
}
logger.WriteLine();
var validationErrors = Validate(benchmarks, logger, config);
if (validationErrors.Any(validationError => validationError.IsCritical))
{
return(Summary.CreateFailed(benchmarks, title, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors));
}
var globalChronometer = Chronometer.Start();
var reports = new List <BenchmarkReport>();
var buildResults = BuildInParallel(logger, rootArtifactsFolderPath, toolchainProvider, resolver, benchmarks, config, ref globalChronometer);
foreach (var benchmark in benchmarks)
{
var buildResult = buildResults[benchmark];
if (!config.KeepBenchmarkFiles)
{
artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup);
}
if (buildResult.IsBuildSuccess)
{
var report = RunCore(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver, buildResult);
reports.Add(report);
if (report.GetResultRuns().Any())
{
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
}
}
else
{
reports.Add(new BenchmarkReport(benchmark, buildResult, buildResult, null, null, default));
if (buildResult.GenerateException != null)
{
logger.WriteLineError($"// Generate Exception: {buildResult.GenerateException.Message}");
}
if (buildResult.BuildException != null)
{
logger.WriteLineError($"// Build Exception: {buildResult.BuildException.Message}");
}
}
logger.WriteLine();
}
var clockSpan = globalChronometer.GetElapsed();
var summary = new Summary(title, reports, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors);
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.WriteLine();
logger.WriteLineHeader("// * Export *");
var currentDirectory = Directory.GetCurrentDirectory();
foreach (var file in config.GetCompositeExporter().ExportToFiles(summary, logger))
{
logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}");
}
logger.WriteLine();
logger.WriteLineHeader("// * Detailed results *");
// TODO: make exporter
foreach (var report in reports)
{
logger.WriteLineInfo(report.Benchmark.DisplayInfo);
logger.WriteLineStatistic($"Runtime = {report.GetRuntimeInfo()}; GC = {report.GetGcInfo()}");
var resultRuns = report.GetResultRuns();
if (resultRuns.IsEmpty())
{
logger.WriteLineError("There are not any results runs");
}
else
{
logger.WriteLineStatistic(resultRuns.GetStatistics().ToTimeStr());
}
logger.WriteLine();
}
LogTotalTime(logger, clockSpan.GetTimeSpan());
logger.WriteLine();
logger.WriteLineHeader("// * Summary *");
MarkdownExporter.Console.ExportToLog(summary, logger);
// TODO: make exporter
ConclusionHelper.Print(logger, config.GetCompositeAnalyser().Analyse(summary).ToList());
// TODO: move to conclusions
var columnWithLegends = summary.Table.Columns.Select(c => c.OriginalColumn).Where(c => !string.IsNullOrEmpty(c.Legend)).ToList();
var effectiveTimeUnit = summary.Table.EffectiveSummaryStyle.TimeUnit;
if (columnWithLegends.Any() || effectiveTimeUnit != null)
{
logger.WriteLine();
logger.WriteLineHeader("// * Legends *");
int maxNameWidth = 0;
if (columnWithLegends.Any())
{
maxNameWidth = Math.Max(maxNameWidth, columnWithLegends.Select(c => c.ColumnName.Length).Max());
}
if (effectiveTimeUnit != null)
{
maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.Name.Length + 2);
}
foreach (var column in columnWithLegends)
{
logger.WriteLineHint($" {column.ColumnName.PadRight(maxNameWidth, ' ')} : {column.Legend}");
}
if (effectiveTimeUnit != null)
{
logger.WriteLineHint($" {("1 " + effectiveTimeUnit.Name).PadRight(maxNameWidth, ' ')} :" +
$" 1 {effectiveTimeUnit.Description} ({TimeUnit.Convert(1, effectiveTimeUnit, TimeUnit.Second).ToStr("0.#########")} sec)");
}
}
if (config.GetDiagnosers().Any())
{
logger.WriteLine();
config.GetCompositeDiagnoser().DisplayResults(logger);
}
logger.WriteLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return(summary);
}
private static Summary Run(BenchmarkRunInfo benchmarkRunInfo,
Dictionary <BenchmarkCase, (BenchmarkId benchmarkId, BuildResult buildResult)> buildResults,
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo)
{
using (DirtyAssemblyResolveHelper.Create())
return(RunWithExceptionHandling(() => RunWithDirtyAssemblyResolveHelper(new[] { benchmarkRunInfo }).Single()));
}
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, bool summaryPerType)
=> Run(new[] { benchmarkRunInfo }, benchmarkRunInfo.Config, summaryPerType).Single();
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo) => Run(new[] { benchmarkRunInfo }, benchmarkRunInfo.Config).Single();
public static Summary Run(BenchmarkRunInfo benchmarks) => BenchmarkRunnerCore.Run(benchmarks, ToolchainExtensions.GetToolchain);