private void FixConfigLoggers(ManualCompetitionConfig competitionConfig) { var runOptions = competitionConfig.Options.RunOptions; var hostLogMode = runOptions.DetailedLogging ? LogFilter.AllMessages : LogFilter.PrefixedOrErrors; competitionConfig.Loggers.Insert(0, CreateHostLogger(hostLogMode)); }
public static void TestInProcessBenchmarkWithValidation() { ResetCounters(); // DONTTOUCH: config SHOULD NOT match the default platform (x64). var config = new ManualCompetitionConfig( CompetitionHelpers.CreateConfig( typeof(InProcessBenchmarkAllCases), new CompetitionFeatures { Platform = Platform.X86 })); config.Add(InProcessValidator.FailOnError); var summary = SelfTestCompetition .Run <InProcessBenchmarkAllCases>(config) .LastRunSummary; Assert.AreEqual(_callCounter, 0); Assert.AreEqual(_afterSetupCounter, 0); Assert.AreEqual(_setupCounter, 0); Assert.AreEqual(_cleanupCounter, 0); Assert.AreEqual(summary?.ValidationErrors.Length, 1); Assert.IsTrue(summary?.ValidationErrors[0].IsCritical); Assert.AreEqual( summary?.ValidationErrors[0].Message, "Job SelfTestConfigX86, EnvMode.Platform was run as X64 (X86 expected). Fix your test runner options."); }
public void Modify(ManualCompetitionConfig competitionConfig) { competitionConfig.Metrics.Add( WellKnownEtwMetrics.FileIoRead); competitionConfig.Metrics.Add( WellKnownEtwMetrics.FileIoWrite); }
private static void FixConfigValidators(ManualCompetitionConfig competitionConfig) { var competitionOptions = competitionConfig.Options; var debugMode = competitionOptions.RunOptions.AllowDebugBuilds; var validators = competitionConfig.Validators; var customToolchain = competitionConfig.Jobs.Any(j => !(j.Infrastructure.Toolchain is InProcessToolchain)); if (!customToolchain && !validators.Any(v => v is InProcessValidator)) { validators.Insert(0, debugMode ? InProcessValidator.DontFailOnError : InProcessValidator.FailOnError); } if (debugMode) { validators.RemoveAll(v => v is JitOptimizationsValidator); } else if (competitionOptions.Adjustments.AdjustMetrics && !validators.OfType <JitOptimizationsValidator>().Any()) { validators.Insert(0, JitOptimizationsValidator.FailOnError); } Code.BugIf( validators.OfType <RunStateSlots>().Any(), "The config validators should not contain instances of RunStateSlots."); // DONTTOUCH: the RunStateSlots should be first in the chain. validators.Insert(0, new RunStateSlots()); }
private void FixConfigExporters(ManualCompetitionConfig competitionConfig) { // HACK: shuts up the ConfigValidator if (competitionConfig.Exporters.Count == 0) { competitionConfig.Exporters.Add(new StubExporter()); } }
private void FixCompetitionConfig(ManualCompetitionConfig competitionConfig) { // DONTTOUCH: call order is important. FixConfigJobs(competitionConfig); FixConfigLoggers(competitionConfig); FixConfigValidators(competitionConfig); FixConfigAnalysers(competitionConfig); FixConfigMetrics(competitionConfig); FixConfigDuplicates(competitionConfig); }
public void RunSensitivityPerfTests() { // The test could fail with "too fast" warning, it's ok var overrideConfig = new ManualCompetitionConfig(RunConfig) { ReportWarningsAsErrors = false }; Competition.Run(this, overrideConfig); }
private static void FixConfigMetrics(ManualCompetitionConfig competitionConfig) { var metrics = competitionConfig.Metrics; competitionConfig.Add( metrics.Select(m => m.GetColumnProvider()).Where(c => c != null).ToArray()); competitionConfig.Add( metrics.Select(m => m.GetDiagnosers()).SelectMany(d => d).ToArray()); }
/// <summary>Updates competition config.</summary> /// <param name="competitionConfig">The competition config.</param> public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.ApplyModifier( new Job { Run = { LaunchCount = 1, WarmupCount = 256, TargetCount = 512 } });
public void RunOpsCountNotSensitivePerfTests() { // The test should fail with warning! var overrideConfig = new ManualCompetitionConfig(RunConfig) { ReportWarningsAsErrors = false }; Competition.Run(this, overrideConfig); }
private static void FixConfigDuplicates(ManualCompetitionConfig competitionConfig) { RemoveDuplicates(competitionConfig.Analysers); RemoveDuplicates(competitionConfig.ColumnProviders); RemoveDuplicates(competitionConfig.Diagnosers); RemoveDuplicates(competitionConfig.Exporters); RemoveDuplicates(competitionConfig.Jobs); RemoveDuplicates(competitionConfig.Loggers); RemoveDuplicates(competitionConfig.Metrics, m => m.AttributeType); RemoveDuplicates(competitionConfig.Validators); }
private static ManualCompetitionConfig CreateRunConfigCore() { var result = new ManualCompetitionConfig { RerunIfLimitsFailed = true, ReportWarningsAsErrors = true }; result.Add(BenchmarkDotNet.Configs.DefaultConfig.Instance.GetColumns().ToArray()); result.Add(AppConfigHelpers.GetImportantOnlyLogger(typeof(SelfTestHelpers).Assembly)); return(result); }
/// <summary>Updates competition config.</summary> /// <param name="competitionConfig">The competition config.</param> public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.ApplyModifier( new Job { Run = { LaunchCount = 1, WarmupCount = 10, TargetCount = 10, InvocationCount = 1, UnrollFactor = 1 } });
private static void FixConfigAnalysers(ManualCompetitionConfig competitionConfig) { Code.BugIf( competitionConfig.Analysers.OfType <CompetitionAnalyser>().Any(), "The config analysers should not contain instances of CompetitionAnalyser."); // DONTTOUCH: these should be first analysers in the chain. competitionConfig.Analysers.Insert(0, ValidatorMessagesAnalyser.Instance); competitionConfig.Analysers.Insert(0, CompetitionPreconditionsAnalyser.Instance); // DONTTOUCH: the CompetitionAnnotateAnalyser should be last analyser in the chain. competitionConfig.Analysers.Add(CompetitionAnalyser.Instance); }
/// <summary>Applies <see cref="ICompetitionModifierSource"/> attribute modifiers.</summary> /// <param name="competitionConfig">The competition configuration.</param> /// <param name="metadataSource">The metadata source.</param> protected virtual void ApplyCompetitionModifiers( [NotNull] ManualCompetitionConfig competitionConfig, [CanBeNull] ICustomAttributeProvider metadataSource) { if (metadataSource != null) { foreach (var modifierSource in metadataSource .GetMetadataAttributes <ICompetitionModifierSource>() .Reverse()) { modifierSource.Modifier.Modify(competitionConfig); } } }
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.ApplyModifier( new CompetitionOptions() { Adjustments = { AdjustMetrics = false }, Annotations = { IgnoreExistingAnnotations = true, PreviousRunLogUri = _previousRunLogUri } });
protected virtual ManualCompetitionConfig CreateEmptyConfig( [CanBeNull] ICustomAttributeProvider metadataSource, [NotNull] CompetitionFeatures competitionFeatures) { var result = new ManualCompetitionConfig(BenchmarkDotNet.Configs.DefaultConfig.Instance); // DONTTOUCH: competition should not use default // Jobs, Diagnosers, Exporters and Loggers. // These omitted intentionally. result.Jobs.Clear(); result.Diagnosers.Clear(); result.Exporters.Clear(); result.Loggers.Clear(); // TODO: better columns. // TODO: columns for options. result.ColumnProviders.Clear(); result.Add(competitionFeatures.TroubleshootingMode ? TroubleshootingModeColumns : DefaultColumns); var targetAssembly = GetAssembly(metadataSource); if (targetAssembly != null) { if (competitionFeatures.TroubleshootingMode) { result.Add( GetImportantInfoLogger(targetAssembly), GetDetailedLogger(targetAssembly)); result.Add(Exporters.CsvTimingsExporter.Default); } else { if (competitionFeatures.ImportantInfoLogger || competitionFeatures.ContinuousIntegrationMode) { result.Add(GetImportantInfoLogger(targetAssembly)); } if (competitionFeatures.DetailedLogger) { result.Add(GetDetailedLogger(targetAssembly)); } } } result.Add(DefaultMetrics.ToArray()); return(result); }
private ICompetitionConfig CreateBenchmarkConfig( [NotNull] Type benchmarkType, ICompetitionConfig competitionConfig, CompetitionFeatures competitionFeatures) { // ReSharper disable once UseObjectOrCollectionInitializer var result = new ManualCompetitionConfig( competitionConfig ?? CompetitionConfigFactory.FindFactoryAndCreate(benchmarkType, competitionFeatures)); InitCompetitionConfigOverride(result); FixCompetitionConfig(result); return(result.AsReadOnly()); }
public static void CompetitionTooSlowOk() { var overrideConfig = new ManualCompetitionConfig(SelfTestConfig) { AllowLongRunningBenchmarks = true }; var runState = SelfTestCompetition.Run <TooSlowBenchmark>(overrideConfig); var messages = runState.GetMessages(); var summary = runState.LastRunSummary; Assert.AreEqual(summary?.ValidationErrors.Length, 0); Assert.AreEqual(runState.RunNumber, 1); Assert.AreEqual(runState.RunsLeft, 0); Assert.AreEqual(runState.RunLimitExceeded, false); Assert.AreEqual(runState.LooksLikeLastRun, true); Assert.AreEqual(messages.Length, 0); }
/// <summary>Applies <see cref="IConfigSource"/> attribute modifiers.</summary> /// <param name="competitionConfig">The competition configuration.</param> /// <param name="metadataSource">The metadata source.</param> protected virtual void ApplyConfigSources(ManualCompetitionConfig competitionConfig, ICustomAttributeProvider metadataSource) { var config = new ManualConfig(); var add = false; if (metadataSource != null) { foreach (var modifierSource in metadataSource .GetMetadataAttributes <IConfigSource>() .Reverse()) { config = ManualConfig.Union(config, modifierSource.Config); add = true; } } if (add) { competitionConfig.Add(config); } }
/// <summary>Helper for custom configs: configuration that should be used for new performance tests.</summary> /// <param name="job">The job for the config. Default one will be used if <c>null</c>.</param> /// <returns>Configuration that should be used for new performance tests.</returns> public static ManualCompetitionConfig CreateDefaultConfig(IJob job = null) { var defaultConfig = BenchmarkDotNet.Configs.DefaultConfig.Instance; var result = new ManualCompetitionConfig { RerunIfLimitsFailed = true, KeepBenchmarkFiles = defaultConfig.KeepBenchmarkFiles }; result.Add(defaultConfig.GetColumns().ToArray()); result.Add(defaultConfig.GetValidators().ToArray()); result.Add(defaultConfig.GetAnalysers().ToArray()); result.Add(defaultConfig.GetDiagnosers().ToArray()); result.Set(defaultConfig.GetOrderProvider()); result.Add(job ?? CreateDefaultJob()); return(result); }
private static void FixConfigJobs(ManualCompetitionConfig competitionConfig) { var jobs = competitionConfig.Jobs; for (var i = 0; i < jobs.Count; i++) { var job = jobs[i]; if (job.Infrastructure.Toolchain == null) { var id = job.HasValue(CharacteristicObject.IdCharacteristic) ? job.Id : null; jobs[i] = job .With(InProcessToolchain.Instance) .WithId(id); } } if (competitionConfig.Jobs.Count == 0) { competitionConfig.Add(CompetitionConfigFactory.DefaultJob); } }
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Add(StatisticColumn.Skewness);
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Add(BaselineScaledColumn.WelchTTestPValue);
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Add(StatisticColumn.Kurtosis);
/// <summary>Customize competition config.</summary> /// <param name="competitionConfig">The competition configuration.</param> protected override void InitCompetitionConfigOverride(ManualCompetitionConfig competitionConfig) { base.InitCompetitionConfigOverride(competitionConfig); competitionConfig.Loggers.RemoveAll(l => l is ConsoleLogger); }
/// <summary>Completes competition config creation.</summary> /// <param name="competitionConfig">Current competition config.</param> /// <returns>Read-only competition config.</returns> protected virtual ICompetitionConfig CompleteConfig( [NotNull] ManualCompetitionConfig competitionConfig) => competitionConfig.AsReadOnly();
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Metrics.Add(WellKnownEtwMetrics.ClrExceptions);
public void Modify(ManualCompetitionConfig competitionConfig) { competitionConfig.Metrics.Add(WellKnownMetrics.ExpectedTime); competitionConfig.Metrics.Add(WellKnownMetrics.GcAllocations); }
/// <summary>Customize competition config.</summary> /// <param name="competitionConfig">The competition configuration.</param> protected virtual void InitCompetitionConfigOverride(ManualCompetitionConfig competitionConfig) { }