private void FixConfigLoggers(ManualCompetitionConfig competitionConfig)
{
var runOptions = competitionConfig.Options.RunOptions;
var hostLogMode = runOptions.DetailedLogging ? LogFilter.AllMessages : LogFilter.PrefixedOrErrors;
competitionConfig.Loggers.Insert(0, CreateHostLogger(hostLogMode));
}
public static void TestInProcessBenchmarkWithValidation()
{
ResetCounters();
// DONTTOUCH: config SHOULD NOT match the default platform (x64).
var config = new ManualCompetitionConfig(
CompetitionHelpers.CreateConfig(
typeof(InProcessBenchmarkAllCases),
new CompetitionFeatures
{
Platform = Platform.X86
}));
config.Add(InProcessValidator.FailOnError);
var summary = SelfTestCompetition
.Run <InProcessBenchmarkAllCases>(config)
.LastRunSummary;
Assert.AreEqual(_callCounter, 0);
Assert.AreEqual(_afterSetupCounter, 0);
Assert.AreEqual(_setupCounter, 0);
Assert.AreEqual(_cleanupCounter, 0);
Assert.AreEqual(summary?.ValidationErrors.Length, 1);
Assert.IsTrue(summary?.ValidationErrors[0].IsCritical);
Assert.AreEqual(
summary?.ValidationErrors[0].Message,
"Job SelfTestConfigX86, EnvMode.Platform was run as X64 (X86 expected). Fix your test runner options.");
}
public void Modify(ManualCompetitionConfig competitionConfig)
{
competitionConfig.Metrics.Add(
WellKnownEtwMetrics.FileIoRead);
competitionConfig.Metrics.Add(
WellKnownEtwMetrics.FileIoWrite);
}
private static void FixConfigValidators(ManualCompetitionConfig competitionConfig)
{
var competitionOptions = competitionConfig.Options;
var debugMode = competitionOptions.RunOptions.AllowDebugBuilds;
var validators = competitionConfig.Validators;
var customToolchain = competitionConfig.Jobs.Any(j => !(j.Infrastructure.Toolchain is InProcessToolchain));
if (!customToolchain &&
!validators.Any(v => v is InProcessValidator))
{
validators.Insert(0, debugMode ? InProcessValidator.DontFailOnError : InProcessValidator.FailOnError);
}
if (debugMode)
{
validators.RemoveAll(v => v is JitOptimizationsValidator);
}
else if (competitionOptions.Adjustments.AdjustMetrics &&
!validators.OfType <JitOptimizationsValidator>().Any())
{
validators.Insert(0, JitOptimizationsValidator.FailOnError);
}
Code.BugIf(
validators.OfType <RunStateSlots>().Any(),
"The config validators should not contain instances of RunStateSlots.");
// DONTTOUCH: the RunStateSlots should be first in the chain.
validators.Insert(0, new RunStateSlots());
}
private void FixConfigExporters(ManualCompetitionConfig competitionConfig)
{
// HACK: shuts up the ConfigValidator
if (competitionConfig.Exporters.Count == 0)
{
competitionConfig.Exporters.Add(new StubExporter());
}
}
private void FixCompetitionConfig(ManualCompetitionConfig competitionConfig)
{
// DONTTOUCH: call order is important.
FixConfigJobs(competitionConfig);
FixConfigLoggers(competitionConfig);
FixConfigValidators(competitionConfig);
FixConfigAnalysers(competitionConfig);
FixConfigMetrics(competitionConfig);
FixConfigDuplicates(competitionConfig);
}
public void RunSensitivityPerfTests()
{
// The test could fail with "too fast" warning, it's ok
var overrideConfig = new ManualCompetitionConfig(RunConfig)
{
ReportWarningsAsErrors = false
};
Competition.Run(this, overrideConfig);
}
private static void FixConfigMetrics(ManualCompetitionConfig competitionConfig)
{
var metrics = competitionConfig.Metrics;
competitionConfig.Add(
metrics.Select(m => m.GetColumnProvider()).Where(c => c != null).ToArray());
competitionConfig.Add(
metrics.Select(m => m.GetDiagnosers()).SelectMany(d => d).ToArray());
}
/// <summary>Updates competition config.</summary>
/// <param name="competitionConfig">The competition config.</param>
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.ApplyModifier(
new Job
{
Run =
{
LaunchCount = 1,
WarmupCount = 256,
TargetCount = 512
}
});
public void RunOpsCountNotSensitivePerfTests()
{
// The test should fail with warning!
var overrideConfig = new ManualCompetitionConfig(RunConfig)
{
ReportWarningsAsErrors = false
};
Competition.Run(this, overrideConfig);
}
private static void FixConfigDuplicates(ManualCompetitionConfig competitionConfig)
{
RemoveDuplicates(competitionConfig.Analysers);
RemoveDuplicates(competitionConfig.ColumnProviders);
RemoveDuplicates(competitionConfig.Diagnosers);
RemoveDuplicates(competitionConfig.Exporters);
RemoveDuplicates(competitionConfig.Jobs);
RemoveDuplicates(competitionConfig.Loggers);
RemoveDuplicates(competitionConfig.Metrics, m => m.AttributeType);
RemoveDuplicates(competitionConfig.Validators);
}
private static ManualCompetitionConfig CreateRunConfigCore()
{
var result = new ManualCompetitionConfig
{
RerunIfLimitsFailed = true,
ReportWarningsAsErrors = true
};
result.Add(BenchmarkDotNet.Configs.DefaultConfig.Instance.GetColumns().ToArray());
result.Add(AppConfigHelpers.GetImportantOnlyLogger(typeof(SelfTestHelpers).Assembly));
return(result);
}
/// <summary>Updates competition config.</summary>
/// <param name="competitionConfig">The competition config.</param>
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.ApplyModifier(
new Job
{
Run =
{
LaunchCount = 1,
WarmupCount = 10,
TargetCount = 10,
InvocationCount = 1,
UnrollFactor = 1
}
});
private static void FixConfigAnalysers(ManualCompetitionConfig competitionConfig)
{
Code.BugIf(
competitionConfig.Analysers.OfType <CompetitionAnalyser>().Any(),
"The config analysers should not contain instances of CompetitionAnalyser.");
// DONTTOUCH: these should be first analysers in the chain.
competitionConfig.Analysers.Insert(0, ValidatorMessagesAnalyser.Instance);
competitionConfig.Analysers.Insert(0, CompetitionPreconditionsAnalyser.Instance);
// DONTTOUCH: the CompetitionAnnotateAnalyser should be last analyser in the chain.
competitionConfig.Analysers.Add(CompetitionAnalyser.Instance);
}
/// <summary>Applies <see cref="ICompetitionModifierSource"/> attribute modifiers.</summary>
/// <param name="competitionConfig">The competition configuration.</param>
/// <param name="metadataSource">The metadata source.</param>
protected virtual void ApplyCompetitionModifiers(
[NotNull] ManualCompetitionConfig competitionConfig,
[CanBeNull] ICustomAttributeProvider metadataSource)
{
if (metadataSource != null)
{
foreach (var modifierSource in metadataSource
.GetMetadataAttributes <ICompetitionModifierSource>()
.Reverse())
{
modifierSource.Modifier.Modify(competitionConfig);
}
}
}
public void Modify(ManualCompetitionConfig competitionConfig) =>
competitionConfig.ApplyModifier(
new CompetitionOptions()
{
Adjustments =
{
AdjustMetrics = false
},
Annotations =
{
IgnoreExistingAnnotations = true,
PreviousRunLogUri = _previousRunLogUri
}
});
protected virtual ManualCompetitionConfig CreateEmptyConfig(
[CanBeNull] ICustomAttributeProvider metadataSource,
[NotNull] CompetitionFeatures competitionFeatures)
{
var result = new ManualCompetitionConfig(BenchmarkDotNet.Configs.DefaultConfig.Instance);
// DONTTOUCH: competition should not use default
// Jobs, Diagnosers, Exporters and Loggers.
// These omitted intentionally.
result.Jobs.Clear();
result.Diagnosers.Clear();
result.Exporters.Clear();
result.Loggers.Clear();
// TODO: better columns.
// TODO: columns for options.
result.ColumnProviders.Clear();
result.Add(competitionFeatures.TroubleshootingMode ? TroubleshootingModeColumns : DefaultColumns);
var targetAssembly = GetAssembly(metadataSource);
if (targetAssembly != null)
{
if (competitionFeatures.TroubleshootingMode)
{
result.Add(
GetImportantInfoLogger(targetAssembly),
GetDetailedLogger(targetAssembly));
result.Add(Exporters.CsvTimingsExporter.Default);
}
else
{
if (competitionFeatures.ImportantInfoLogger || competitionFeatures.ContinuousIntegrationMode)
{
result.Add(GetImportantInfoLogger(targetAssembly));
}
if (competitionFeatures.DetailedLogger)
{
result.Add(GetDetailedLogger(targetAssembly));
}
}
}
result.Add(DefaultMetrics.ToArray());
return(result);
}
private ICompetitionConfig CreateBenchmarkConfig(
[NotNull] Type benchmarkType,
ICompetitionConfig competitionConfig,
CompetitionFeatures competitionFeatures)
{
// ReSharper disable once UseObjectOrCollectionInitializer
var result = new ManualCompetitionConfig(
competitionConfig ??
CompetitionConfigFactory.FindFactoryAndCreate(benchmarkType, competitionFeatures));
InitCompetitionConfigOverride(result);
FixCompetitionConfig(result);
return(result.AsReadOnly());
}
public static void CompetitionTooSlowOk()
{
var overrideConfig = new ManualCompetitionConfig(SelfTestConfig)
{
AllowLongRunningBenchmarks = true
};
var runState = SelfTestCompetition.Run <TooSlowBenchmark>(overrideConfig);
var messages = runState.GetMessages();
var summary = runState.LastRunSummary;
Assert.AreEqual(summary?.ValidationErrors.Length, 0);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 0);
}
/// <summary>Applies <see cref="IConfigSource"/> attribute modifiers.</summary>
/// <param name="competitionConfig">The competition configuration.</param>
/// <param name="metadataSource">The metadata source.</param>
protected virtual void ApplyConfigSources(ManualCompetitionConfig competitionConfig, ICustomAttributeProvider metadataSource)
{
var config = new ManualConfig();
var add = false;
if (metadataSource != null)
{
foreach (var modifierSource in metadataSource
.GetMetadataAttributes <IConfigSource>()
.Reverse())
{
config = ManualConfig.Union(config, modifierSource.Config);
add = true;
}
}
if (add)
{
competitionConfig.Add(config);
}
}
/// <summary>Helper for custom configs: configuration that should be used for new performance tests.</summary>
/// <param name="job">The job for the config. Default one will be used if <c>null</c>.</param>
/// <returns>Configuration that should be used for new performance tests.</returns>
public static ManualCompetitionConfig CreateDefaultConfig(IJob job = null)
{
var defaultConfig = BenchmarkDotNet.Configs.DefaultConfig.Instance;
var result = new ManualCompetitionConfig
{
RerunIfLimitsFailed = true,
KeepBenchmarkFiles = defaultConfig.KeepBenchmarkFiles
};
result.Add(defaultConfig.GetColumns().ToArray());
result.Add(defaultConfig.GetValidators().ToArray());
result.Add(defaultConfig.GetAnalysers().ToArray());
result.Add(defaultConfig.GetDiagnosers().ToArray());
result.Set(defaultConfig.GetOrderProvider());
result.Add(job ?? CreateDefaultJob());
return(result);
}
private static void FixConfigJobs(ManualCompetitionConfig competitionConfig)
{
var jobs = competitionConfig.Jobs;
for (var i = 0; i < jobs.Count; i++)
{
var job = jobs[i];
if (job.Infrastructure.Toolchain == null)
{
var id = job.HasValue(CharacteristicObject.IdCharacteristic) ? job.Id : null;
jobs[i] = job
.With(InProcessToolchain.Instance)
.WithId(id);
}
}
if (competitionConfig.Jobs.Count == 0)
{
competitionConfig.Add(CompetitionConfigFactory.DefaultJob);
}
}
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Add(StatisticColumn.Skewness);
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Add(BaselineScaledColumn.WelchTTestPValue);
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Add(StatisticColumn.Kurtosis);
/// <summary>Customize competition config.</summary>
/// <param name="competitionConfig">The competition configuration.</param>
protected override void InitCompetitionConfigOverride(ManualCompetitionConfig competitionConfig)
{
base.InitCompetitionConfigOverride(competitionConfig);
competitionConfig.Loggers.RemoveAll(l => l is ConsoleLogger);
}
/// <summary>Completes competition config creation.</summary>
/// <param name="competitionConfig">Current competition config.</param>
/// <returns>Read-only competition config.</returns>
protected virtual ICompetitionConfig CompleteConfig(
[NotNull] ManualCompetitionConfig competitionConfig) =>
competitionConfig.AsReadOnly();
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Metrics.Add(WellKnownEtwMetrics.ClrExceptions);
public void Modify(ManualCompetitionConfig competitionConfig)
{
competitionConfig.Metrics.Add(WellKnownMetrics.ExpectedTime);
competitionConfig.Metrics.Add(WellKnownMetrics.GcAllocations);
}
/// <summary>Customize competition config.</summary>
/// <param name="competitionConfig">The competition configuration.</param>
protected virtual void InitCompetitionConfigOverride(ManualCompetitionConfig competitionConfig)
{
}
