internal static Summary CreateFailed(Benchmark[] benchmarks, string title, HostEnvironmentInfo hostEnvironmentInfo, IConfig config, string resultsDirectoryPath, ValidationError[] validationErrors)
{
return(new Summary(title, hostEnvironmentInfo, config, resultsDirectoryPath, TimeSpan.Zero, validationErrors, benchmarks, new BenchmarkReport[0]));
}
public static Summary Run(Benchmark[] benchmarks, ILogger logger, string title, IConfig config, string rootArtifactsFolderPath, Func <Job, IToolchain> toolchainProvider, IResolver resolver)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
{
logger.WriteLineInfo($"// {benchmark.DisplayInfo}");
}
logger.WriteLine();
var validationErrors = Validate(benchmarks, logger, config);
if (validationErrors.Any(validationError => validationError.IsCritical))
{
return(Summary.CreateFailed(benchmarks, title, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors));
}
var globalChronometer = Chronometer.Start();
var reports = new List <BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
var report = Run(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver);
reports.Add(report);
if (report.GetResultRuns().Any())
{
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
}
logger.WriteLine();
}
var clockSpan = globalChronometer.Stop();
var summary = new Summary(title, reports, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors);
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.WriteLine();
logger.WriteLineHeader("// * Export *");
var currentDirectory = Directory.GetCurrentDirectory();
foreach (var file in config.GetCompositeExporter().ExportToFiles(summary, logger))
{
logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}");
}
logger.WriteLine();
logger.WriteLineHeader("// * Detailed results *");
// TODO: make exporter
foreach (var report in reports)
{
logger.WriteLineInfo(report.Benchmark.DisplayInfo);
logger.WriteLineStatistic($"Runtime = {report.GetRuntimeInfo()}; GC = {report.GetGcInfo()}");
var resultRuns = report.GetResultRuns();
if (resultRuns.IsEmpty())
{
logger.WriteLineError("There are no any results runs");
}
else
{
logger.WriteLineStatistic(resultRuns.GetStatistics().ToTimeStr());
}
logger.WriteLine();
}
LogTotalTime(logger, clockSpan.GetTimeSpan());
logger.WriteLine();
logger.WriteLineHeader("// * Summary *");
MarkdownExporter.Console.ExportToLog(summary, logger);
// TODO: make exporter
ConclusionHelper.Print(logger, config.GetCompositeAnalyser().Analyse(summary).ToList());
if (config.GetDiagnosers().Any())
{
logger.WriteLine();
config.GetCompositeDiagnoser().DisplayResults(logger);
}
logger.WriteLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return(summary);
}
public HostEnvironmentInfoDto(HostEnvironmentInfo hei) => this.hei = hei;
internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, ImmutableArray <ValidationError> validationErrors) => new Summary(title, ImmutableArray <BenchmarkReport> .Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, validationErrors);
private static bool IsUsingProjectJson() =>
HostEnvironmentInfo.GetCurrent().DotNetCliVersion.Value.Contains("preview") &&
SolutionDirectoryContainsProjectJsonFiles();
internal static Summary NothingToRun(string title, string resultsDirectoryPath, string logFilePath) => new Summary(title, ImmutableArray <BenchmarkReport> .Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, ImmutableArray <ValidationError> .Empty);
public static Summary Run(Benchmark[] benchmarks, ILogger logger, string title, IConfig config, string rootArtifactsFolderPath, Func <Job, IToolchain> toolchainProvider, IResolver resolver)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
{
logger.WriteLineInfo($"// {benchmark.DisplayInfo}");
}
logger.WriteLine();
var validationErrors = Validate(benchmarks, logger, config);
if (validationErrors.Any(validationError => validationError.IsCritical))
{
return(Summary.CreateFailed(benchmarks, title, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors));
}
var globalChronometer = Chronometer.Start();
var reports = new List <BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
var report = Run(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver);
reports.Add(report);
if (report.GetResultRuns().Any())
{
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
}
logger.WriteLine();
}
var clockSpan = globalChronometer.Stop();
var summary = new Summary(title, reports, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors);
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.WriteLine();
logger.WriteLineHeader("// * Export *");
var currentDirectory = Directory.GetCurrentDirectory();
foreach (var file in config.GetCompositeExporter().ExportToFiles(summary, logger))
{
logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}");
}
logger.WriteLine();
logger.WriteLineHeader("// * Detailed results *");
// TODO: make exporter
foreach (var report in reports)
{
logger.WriteLineInfo(report.Benchmark.DisplayInfo);
logger.WriteLineStatistic($"Runtime = {report.GetRuntimeInfo()}; GC = {report.GetGcInfo()}");
var resultRuns = report.GetResultRuns();
if (resultRuns.IsEmpty())
{
logger.WriteLineError("There are no any results runs");
}
else
{
logger.WriteLineStatistic(resultRuns.GetStatistics().ToTimeStr());
}
logger.WriteLine();
}
LogTotalTime(logger, clockSpan.GetTimeSpan());
logger.WriteLine();
logger.WriteLineHeader("// * Summary *");
MarkdownExporter.Console.ExportToLog(summary, logger);
// TODO: make exporter
ConclusionHelper.Print(logger, config.GetCompositeAnalyser().Analyse(summary).ToList());
// TODO: move to conclusions
var columnWithLegends = summary.Table.Columns.Select(c => c.OriginalColumn).Where(c => !string.IsNullOrEmpty(c.Legend)).ToList();
var effectiveTimeUnit = summary.Table.EffectiveSummaryStyle.TimeUnit;
if (columnWithLegends.Any() || effectiveTimeUnit != null)
{
logger.WriteLine();
logger.WriteLineHeader("// * Legends *");
int maxNameWidth = 0;
if (columnWithLegends.Any())
{
maxNameWidth = Math.Max(maxNameWidth, columnWithLegends.Select(c => c.ColumnName.Length).Max());
}
if (effectiveTimeUnit != null)
{
maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.Name.Length + 2);
}
foreach (var column in columnWithLegends)
{
logger.WriteLineHint($" {column.ColumnName.PadRight(maxNameWidth, ' ')} : {column.Legend}");
}
if (effectiveTimeUnit != null)
{
logger.WriteLineHint($" {("1 " + effectiveTimeUnit.Name).PadRight(maxNameWidth, ' ')} :" +
$" 1 {effectiveTimeUnit.Description} ({TimeUnit.Convert(1, effectiveTimeUnit, TimeUnit.Second).ToStr("0.#########")} sec)");
}
}
if (config.GetDiagnosers().Any())
{
logger.WriteLine();
config.GetCompositeDiagnoser().DisplayResults(logger);
}
logger.WriteLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return(summary);
}
public static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos, IConfig commonSettingsConfig)
{
var resolver = DefaultResolver;
var artifactsToCleanup = new List <string>();
var title = GetTitle(benchmarkRunInfos);
var rootArtifactsFolderPath = (commonSettingsConfig?.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath).CreateIfNotExists();
using (var logStreamWriter = Portability.StreamWriter.FromPath(Path.Combine(rootArtifactsFolderPath, title + ".log")))
{
var logger = new CompositeLogger(commonSettingsConfig.GetCompositeLogger(), new StreamLogger(logStreamWriter));
var supportedBenchmarks = GetSupportedBenchmarks(benchmarkRunInfos, logger, resolver);
if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any()))
{
return new [] { Summary.CreateFailed(
supportedBenchmarks.SelectMany(b => b.BenchmarksCases).ToArray(),
title, HostEnvironmentInfo.GetCurrent(), commonSettingsConfig, GetResultsFolderPath(rootArtifactsFolderPath), Array.Empty <ValidationError>()) }
}
;
var validationErrors = Validate(supportedBenchmarks, logger);
if (validationErrors.Any(validationError => validationError.IsCritical))
{
return new [] { Summary.CreateFailed(
supportedBenchmarks.SelectMany(b => b.BenchmarksCases).ToArray(),
title, HostEnvironmentInfo.GetCurrent(), commonSettingsConfig, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors) }
}
;
var buildPartitions = BenchmarkPartitioner.CreateForBuild(supportedBenchmarks, resolver);
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
var globalChronometer = Chronometer.Start();
var buildResults = BuildInParallel(logger, rootArtifactsFolderPath, buildPartitions, ref globalChronometer);
try
{
var results = new List <Summary>();
var benchmarkToBuildResult = buildResults
.SelectMany(buildResult => buildResult.Key.Benchmarks.Select(buildInfo => (buildInfo.BenchmarkCase, buildInfo.Id, buildResult.Value)))
.ToDictionary(info => info.BenchmarkCase, info => (info.Id, info.Value));
foreach (var benchmarkRunInfo in supportedBenchmarks) // we run them in the old order now using the new build artifacts
{
var runChronometer = Chronometer.Start();
var summary = Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, logger, artifactsToCleanup, rootArtifactsFolderPath, ref runChronometer);
if (commonSettingsConfig.SummaryPerType)
{
PrintSummary(logger, benchmarkRunInfo.Config, summary);
}
LogTotalTime(logger, runChronometer.GetElapsed().GetTimeSpan(), summary.GetNumberOfExecutedBenchmarks(), message: "Run time");
logger.WriteLine();
results.Add(summary);
}
if (!commonSettingsConfig.SummaryPerType)
{
var joinedSummary = Summary.Join(results, commonSettingsConfig, globalChronometer.GetElapsed());
PrintSummary(logger, commonSettingsConfig, joinedSummary);
results.Clear();
results.Add(joinedSummary);
}
return(results.ToArray());
}
finally
{
logger.WriteLineHeader("// * Artifacts cleanup *");
Cleanup(new HashSet <string>(artifactsToCleanup.Distinct()));
}
}
}
public GcMode CreateDefaultValue() => new GcMode
{
Concurrent = HostEnvironmentInfo.GetCurrent().IsConcurrentGC,
Server = HostEnvironmentInfo.GetCurrent().IsServerGC
};
public ComputerSpecifications()
{
_hostEnvironmentInfo = HostEnvironmentInfo.GetCurrent();
}
