private static bool CheckSetup(ResultAnalysis analysis) { var summary = analysis.Summary; // DONTTOUCH: DO NOT add return into if clauses. // All conditions should be checked if (summary.HasCriticalValidationErrors) { analysis.WriteExecutionErrorMessage("Summary has validation errors."); } if (!summary.HasCriticalValidationErrors && summary.Benchmarks.IsNullOrEmpty()) { analysis.WriteSetupErrorMessage( "Nothing to check as there is no methods in benchmark.", $"Apply one of {nameof(CompetitionBenchmarkAttribute)}, {nameof(CompetitionBaselineAttribute)} or {nameof(BenchmarkAttribute)} to the benchmark methods."); } if (summary.Config.GetJobs().Skip(1).Any()) { analysis.WriteSetupErrorMessage( "Benchmark configuration includes multiple jobs. " + "This is not supported as there's no way to store metric annotations individually per each job.", "Ensure that the config contains only one job."); } if (summary.Benchmarks.Select(b => b.Parameters).Distinct().Skip(1).Any()) { analysis.WriteInfoMessage( "Benchmark configuration includes multiple parameters. " + "Note that results for each parameter set will be merged."); } CheckMembers(analysis); CheckMetrics(analysis); return(analysis.SafeToContinue); }
private static void CheckExecution(ResultAnalysis analysis) { var summary = analysis.Summary; // DONTTOUCH: DO NOT add return into if clauses. // All conditions should be checked var benchmarksWithReports = summary.Reports .Where(r => r.ExecuteResults.Any()) .Select(r => r.Benchmark); var benchMissing = summary.GetSummaryOrderBenchmarks() .Except(benchmarksWithReports) .Select(b => b.Target.MethodDisplayInfo) .Distinct(). ToArray(); if (benchMissing.Any()) { var benchmarks = benchMissing.Length == 1 ? "benchmark" : "benchmarks"; analysis.WriteExecutionErrorMessage( $"No result reports for {benchmarks}: {benchMissing.Join(", ")}.", "Ensure that benchmarks were run successfully and did not throw any exceptions."); } var checksMode = analysis.Options.Checks; if (checksMode.CheckMetrics) { var timeUnits = MetricUnitScale.FromEnumValues(typeof(TimeUnit)); if (checksMode.TooFastBenchmarkLimit > TimeSpan.Zero) { var tooFastReports = GetTargetNames( analysis, r => r.GetResultRuns().Average(run => run.Nanoseconds) < checksMode.TooFastBenchmarkLimit.TotalNanoseconds()); if (tooFastReports.Any()) { var benchmarks = tooFastReports.Length == 1 ? "Benchmark" : "Benchmarks"; var time = checksMode.TooFastBenchmarkLimit .TotalNanoseconds() .ToString(timeUnits); analysis.AddWarningConclusion( $"{benchmarks} {tooFastReports.Join(", ")}: measured run time is less than {time}. " + "Timings are imprecise as they are too close to the timer resolution.", $"Timing limit for this warning is configured via {CompetitionCheckMode.TooFastBenchmarkLimitCharacteristic.FullId}."); } } if (checksMode.LongRunningBenchmarkLimit > TimeSpan.Zero) { var tooSlowReports = GetTargetNames( analysis, r => r.GetResultRuns().Average(run => run.Nanoseconds) > checksMode.LongRunningBenchmarkLimit.TotalNanoseconds()); if (tooSlowReports.Any()) { var benchmarks = tooSlowReports.Length == 1 ? "Benchmark" : "Benchmarks"; var time = checksMode.LongRunningBenchmarkLimit .TotalNanoseconds() .ToString(timeUnits); analysis.AddWarningConclusion( $"{benchmarks} {string.Join(", ", tooSlowReports)}: measured run time is greater than {time}. " + "There's a risk the peak timings were hidden by averages. " + "Consider to reduce the number of iterations performed per each measurement.", $"Timing limit for this warning is configured via {CompetitionCheckMode.LongRunningBenchmarkLimitCharacteristic.FullId}."); } } } }