public override void ExportToLog(Summary summary, ILogger logger) { if (useCodeBlocks) { logger.WriteLine($"```{codeBlocksSyntax}"); } logger = GetRightLogger(logger); logger.WriteLine(); foreach (var infoLine in EnvironmentInfo.GetCurrent().ToFormattedString("Host", true)) { logger.WriteLineInfo(infoLine); } logger.WriteLine(); PrintTable(summary.Table, logger); // TODO: move this logic to an analyser var benchmarksWithTroubles = summary.Reports.Where(r => !r.GetResultRuns().Any()).Select(r => r.Benchmark).ToList(); if (benchmarksWithTroubles.Count > 0) { logger.WriteLine(); logger.WriteLineError("Benchmarks with issues:"); foreach (var benchmarkWithTroubles in benchmarksWithTroubles) { logger.WriteLineError(" " + benchmarkWithTroubles.ShortInfo); } } }
/// <summary>Runs the benchmark.</summary> public void Run() { if (_benchmark == null) { throw new InvalidOperationException("Call Init() first."); } using (BenchmarkHelpers.CaptureConsoleOutput(_output)) { try { FillProperties(_instance, _benchmark); _output.WriteLine(); foreach (var infoLine in EnvironmentInfo.GetCurrent().ToFormattedString()) { _output.WriteLine("// {0}", infoLine); } _output.WriteLine(); new MethodInvokerLight(_job).Invoke(_job, _operationsPerInvoke, _setupAction, _runCallback, _idleCallback); } catch (Exception ex) { _output.WriteLine(ex); throw; } } }
// TODO: check that all diagnosers can be run in-process /// <summary>Proofs that benchmarks' jobs match the environment.</summary> /// <param name="benchmarks">The benchmarks to validate.</param> /// <returns>Enumerable of validation errors.</returns> public IEnumerable <ValidationError> Validate(IList <Benchmark> benchmarks) { var result = new List <ValidationError>(); var env = EnvironmentInfo.GetCurrent(); foreach (var job in benchmarks.GetJobs()) { foreach (var jobProperty in job.AllProperties) { Func <IJob, EnvironmentInfo, string> validationRule; if (_validationRules.TryGetValue(jobProperty.Name, out validationRule)) { var message = validationRule(job, env); if (!string.IsNullOrEmpty(message)) { result.Add( new ValidationError( TreatsWarningsAsErrors, $"Job {job.GetShortInfo()}, property {jobProperty.Name}: {message}")); } } else { result.Add( new ValidationError( false, $"Job {job.GetShortInfo()}, property {jobProperty.Name}: no validation rule specified.")); } } } return(result.ToArray()); }
public override bool IsSupported(Benchmark benchmark, ILogger logger) { if (!EnvironmentInfo.GetCurrent().IsDotNetCliInstalled()) { logger.WriteLineError($"BenchmarkDotNet requires dotnet cli toolchain to be installed, benchmark {benchmark.ShortInfo} will not be executed"); return(false); } return(true); }
public override void ExportToLog(Summary summary, ILogger logger) { logger.Write("<pre><code>"); logger.WriteLine(); foreach (var infoLine in EnvironmentInfo.GetCurrent().ToFormattedString("Host", true)) { logger.WriteLine(infoLine); } logger.Write("</code></pre>"); logger.WriteLine(); PrintTable(summary.Table, logger); }
private void GenerateAppConfigFile(string projectDir, IJob job) { var useLagacyJit = job.Jit == Jit.RyuJit || (job.Jit == Jit.Host && EnvironmentInfo.GetCurrent().HasRyuJit) ? "0" : "1"; var template = ResourceHelper.LoadTemplate( job.Jit == Jit.Host ? "BenchmarkAppConfigEmpty.txt" : "BenchmarkAppConfig.txt"); var content = template. Replace("$UseLagacyJit$", useLagacyJit); string fileName = Path.Combine(projectDir, "app.config"); File.WriteAllText(fileName, content); }
public void Test() { var info = EnvironmentInfo.GetCurrent(); var text = TypeSerializer<EnvironmentInfo>.ToXmlString(info); Assert.IsNotEmpty(text); Console.WriteLine(text); // Console.WriteLine(Environment.WorkingSet); //Console.WriteLine(WindowsIdentity.GetCurrent().Name); //Console.WriteLine(System.Threading.Thread.CurrentPrincipal.Identity.Name); //foreach (var gr in WindowsIdentity.GetCurrent().Groups) //{ // Console.WriteLine(gr.Translate(typeof(NTAccount)).Value); //} }
public override bool IsSupported(Benchmark benchmark, ILogger logger) { if (!EnvironmentInfo.GetCurrent().IsDotNetCliInstalled()) { logger.WriteLineError($"BenchmarkDotNet requires dotnet cli toolchain to be installed, benchmark {benchmark.ShortInfo} will not be executed"); return(false); } if (benchmark.Job.Platform == Platform.X86) { logger.WriteLineError($"Currently dotnet cli toolchain supports only X64 compilation, benchmark {benchmark.ShortInfo} will not be executed"); return(false); } if (benchmark.Job.Jit == Jit.LegacyJit) { logger.WriteLineError($"Currently dotnet cli toolchain supports only RyuJit, benchmark {benchmark.ShortInfo} will not be executed"); return(false); } return(true); }
private ICompetitionConfig PrepareCompetitionConfig( [NotNull] Type benchmarkType, [CanBeNull] ICompetitionConfig competitionConfig) { var result = GetFirstConfig(benchmarkType, competitionConfig); if (result.CompetitionLimitProvider == null) { result.CompetitionLimitProvider = LogNormalLimitProvider.Instance; } if (result.MaxRunsAllowed <= 0) { result.MaxRunsAllowed = DefaultMaxRunsAllowed; } if (EnvironmentInfo.GetCurrent().HasAttachedDebugger) { result.AllowDebugBuilds = true; } return(result.AsReadOnly()); }
private static Summary Run(Benchmark[] benchmarks, ILogger logger, string title, IConfig config, string rootArtifactsFolderPath) { logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); logger.WriteLineInfo("// Found benchmarks:"); foreach (var benchmark in benchmarks) { logger.WriteLineInfo($"// {benchmark.ShortInfo}"); } logger.WriteLine(); var validationErrors = Validate(benchmarks, logger, config); if (validationErrors.Any(validationError => validationError.IsCritical)) { return(Summary.CreateFailed(benchmarks, title, EnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors)); } var globalChronometer = Chronometer.Start(); var reports = new List <BenchmarkReport>(); foreach (var benchmark in benchmarks) { var report = Run(benchmark, logger, config, rootArtifactsFolderPath); reports.Add(report); if (report.GetResultRuns().Any()) { logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr()); } logger.WriteLine(); } var clockSpan = globalChronometer.Stop(); var summary = new Summary(title, reports, EnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors); logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****"); logger.WriteLine(); logger.WriteLineHeader("// * Export *"); var currentDirectory = Directory.GetCurrentDirectory(); foreach (var file in config.GetCompositeExporter().ExportToFiles(summary)) { logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}"); } logger.WriteLine(); logger.WriteLineHeader("// * Detailed results *"); // TODO: make exporter foreach (var report in reports) { logger.WriteLineInfo(report.Benchmark.ShortInfo); logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr()); logger.WriteLine(); } LogTotalTime(logger, clockSpan.GetTimeSpan()); logger.WriteLine(); logger.WriteLineHeader("// * Summary *"); MarkdownExporter.Console.ExportToLog(summary, logger); // TODO: make exporter var warnings = config.GetCompositeAnalyser().Analyse(summary).ToList(); if (warnings.Count > 0) { logger.WriteLine(); logger.WriteLineError("// * Warnings * "); foreach (var warning in warnings) { logger.WriteLineError($"{warning.Message}"); } } if (config.GetDiagnosers().Count() > 0) { logger.WriteLine(); config.GetCompositeDiagnoser().DisplayResults(logger); } logger.WriteLine(); logger.WriteLineHeader("// ***** BenchmarkRunner: End *****"); return(summary); }