public override void ExportToLog(Summary summary, ILogger logger)
{
if (useCodeBlocks)
{
logger.WriteLine($"```{codeBlocksSyntax}");
}
logger = GetRightLogger(logger);
logger.WriteLine();
foreach (var infoLine in EnvironmentInfo.GetCurrent().ToFormattedString("Host", true))
{
logger.WriteLineInfo(infoLine);
}
logger.WriteLine();
PrintTable(summary.Table, logger);
// TODO: move this logic to an analyser
var benchmarksWithTroubles = summary.Reports.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.WriteLine();
logger.WriteLineError("Benchmarks with issues:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
{
logger.WriteLineError(" " + benchmarkWithTroubles.ShortInfo);
}
}
}
/// <summary>Runs the benchmark.</summary>
public void Run()
{
if (_benchmark == null)
{
throw new InvalidOperationException("Call Init() first.");
}
using (BenchmarkHelpers.CaptureConsoleOutput(_output))
{
try
{
FillProperties(_instance, _benchmark);
_output.WriteLine();
foreach (var infoLine in EnvironmentInfo.GetCurrent().ToFormattedString())
{
_output.WriteLine("// {0}", infoLine);
}
_output.WriteLine();
new MethodInvokerLight(_job).Invoke(_job, _operationsPerInvoke, _setupAction, _runCallback, _idleCallback);
}
catch (Exception ex)
{
_output.WriteLine(ex);
throw;
}
}
}
// TODO: check that all diagnosers can be run in-process
/// <summary>Proofs that benchmarks' jobs match the environment.</summary>
/// <param name="benchmarks">The benchmarks to validate.</param>
/// <returns>Enumerable of validation errors.</returns>
public IEnumerable <ValidationError> Validate(IList <Benchmark> benchmarks)
{
var result = new List <ValidationError>();
var env = EnvironmentInfo.GetCurrent();
foreach (var job in benchmarks.GetJobs())
{
foreach (var jobProperty in job.AllProperties)
{
Func <IJob, EnvironmentInfo, string> validationRule;
if (_validationRules.TryGetValue(jobProperty.Name, out validationRule))
{
var message = validationRule(job, env);
if (!string.IsNullOrEmpty(message))
{
result.Add(
new ValidationError(
TreatsWarningsAsErrors,
$"Job {job.GetShortInfo()}, property {jobProperty.Name}: {message}"));
}
}
else
{
result.Add(
new ValidationError(
false,
$"Job {job.GetShortInfo()}, property {jobProperty.Name}: no validation rule specified."));
}
}
}
return(result.ToArray());
}
public override bool IsSupported(Benchmark benchmark, ILogger logger)
{
if (!EnvironmentInfo.GetCurrent().IsDotNetCliInstalled())
{
logger.WriteLineError($"BenchmarkDotNet requires dotnet cli toolchain to be installed, benchmark {benchmark.ShortInfo} will not be executed");
return(false);
}
return(true);
}
public override void ExportToLog(Summary summary, ILogger logger)
{
logger.Write("<pre><code>");
logger.WriteLine();
foreach (var infoLine in EnvironmentInfo.GetCurrent().ToFormattedString("Host", true))
{
logger.WriteLine(infoLine);
}
logger.Write("</code></pre>");
logger.WriteLine();
PrintTable(summary.Table, logger);
}
private void GenerateAppConfigFile(string projectDir, IJob job)
{
var useLagacyJit = job.Jit == Jit.RyuJit ||
(job.Jit == Jit.Host && EnvironmentInfo.GetCurrent().HasRyuJit)
? "0"
: "1";
var template =
ResourceHelper.LoadTemplate(
job.Jit == Jit.Host ? "BenchmarkAppConfigEmpty.txt" : "BenchmarkAppConfig.txt");
var content = template.
Replace("$UseLagacyJit$", useLagacyJit);
string fileName = Path.Combine(projectDir, "app.config");
File.WriteAllText(fileName, content);
}
public void Test()
{
var info = EnvironmentInfo.GetCurrent();
var text = TypeSerializer<EnvironmentInfo>.ToXmlString(info);
Assert.IsNotEmpty(text);
Console.WriteLine(text);
// Console.WriteLine(Environment.WorkingSet);
//Console.WriteLine(WindowsIdentity.GetCurrent().Name);
//Console.WriteLine(System.Threading.Thread.CurrentPrincipal.Identity.Name);
//foreach (var gr in WindowsIdentity.GetCurrent().Groups)
//{
// Console.WriteLine(gr.Translate(typeof(NTAccount)).Value);
//}
}
public override bool IsSupported(Benchmark benchmark, ILogger logger)
{
if (!EnvironmentInfo.GetCurrent().IsDotNetCliInstalled())
{
logger.WriteLineError($"BenchmarkDotNet requires dotnet cli toolchain to be installed, benchmark {benchmark.ShortInfo} will not be executed");
return(false);
}
if (benchmark.Job.Platform == Platform.X86)
{
logger.WriteLineError($"Currently dotnet cli toolchain supports only X64 compilation, benchmark {benchmark.ShortInfo} will not be executed");
return(false);
}
if (benchmark.Job.Jit == Jit.LegacyJit)
{
logger.WriteLineError($"Currently dotnet cli toolchain supports only RyuJit, benchmark {benchmark.ShortInfo} will not be executed");
return(false);
}
return(true);
}
private ICompetitionConfig PrepareCompetitionConfig(
[NotNull] Type benchmarkType,
[CanBeNull] ICompetitionConfig competitionConfig)
{
var result = GetFirstConfig(benchmarkType, competitionConfig);
if (result.CompetitionLimitProvider == null)
{
result.CompetitionLimitProvider = LogNormalLimitProvider.Instance;
}
if (result.MaxRunsAllowed <= 0)
{
result.MaxRunsAllowed = DefaultMaxRunsAllowed;
}
if (EnvironmentInfo.GetCurrent().HasAttachedDebugger)
{
result.AllowDebugBuilds = true;
}
return(result.AsReadOnly());
}
private static Summary Run(Benchmark[] benchmarks, ILogger logger, string title, IConfig config, string rootArtifactsFolderPath)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
{
logger.WriteLineInfo($"// {benchmark.ShortInfo}");
}
logger.WriteLine();
var validationErrors = Validate(benchmarks, logger, config);
if (validationErrors.Any(validationError => validationError.IsCritical))
{
return(Summary.CreateFailed(benchmarks, title, EnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors));
}
var globalChronometer = Chronometer.Start();
var reports = new List <BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
var report = Run(benchmark, logger, config, rootArtifactsFolderPath);
reports.Add(report);
if (report.GetResultRuns().Any())
{
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
}
logger.WriteLine();
}
var clockSpan = globalChronometer.Stop();
var summary = new Summary(title, reports, EnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors);
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.WriteLine();
logger.WriteLineHeader("// * Export *");
var currentDirectory = Directory.GetCurrentDirectory();
foreach (var file in config.GetCompositeExporter().ExportToFiles(summary))
{
logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}");
}
logger.WriteLine();
logger.WriteLineHeader("// * Detailed results *");
// TODO: make exporter
foreach (var report in reports)
{
logger.WriteLineInfo(report.Benchmark.ShortInfo);
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
logger.WriteLine();
}
LogTotalTime(logger, clockSpan.GetTimeSpan());
logger.WriteLine();
logger.WriteLineHeader("// * Summary *");
MarkdownExporter.Console.ExportToLog(summary, logger);
// TODO: make exporter
var warnings = config.GetCompositeAnalyser().Analyse(summary).ToList();
if (warnings.Count > 0)
{
logger.WriteLine();
logger.WriteLineError("// * Warnings * ");
foreach (var warning in warnings)
{
logger.WriteLineError($"{warning.Message}");
}
}
if (config.GetDiagnosers().Count() > 0)
{
logger.WriteLine();
config.GetCompositeDiagnoser().DisplayResults(logger);
}
logger.WriteLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return(summary);
}
