private static void FixConfigMetrics(ManualCompetitionConfig competitionConfig)
{
var metrics = competitionConfig.Metrics;
competitionConfig.Add(
metrics.Select(m => m.GetColumnProvider()).Where(c => c != null).ToArray());
competitionConfig.Add(
metrics.Select(m => m.GetDiagnosers()).SelectMany(d => d).ToArray());
}
private static ManualCompetitionConfig CreateRunConfigCore()
{
var result = new ManualCompetitionConfig
{
RerunIfLimitsFailed = true,
ReportWarningsAsErrors = true
};
result.Add(BenchmarkDotNet.Configs.DefaultConfig.Instance.GetColumns().ToArray());
result.Add(AppConfigHelpers.GetImportantOnlyLogger(typeof(SelfTestHelpers).Assembly));
return(result);
}
public static void TestInProcessBenchmarkWithValidation()
{
ResetCounters();
// DONTTOUCH: config SHOULD NOT match the default platform (x64).
var config = new ManualCompetitionConfig(
CompetitionHelpers.CreateConfig(
typeof(InProcessBenchmarkAllCases),
new CompetitionFeatures
{
Platform = Platform.X86
}));
config.Add(InProcessValidator.FailOnError);
var summary = SelfTestCompetition
.Run <InProcessBenchmarkAllCases>(config)
.LastRunSummary;
Assert.AreEqual(_callCounter, 0);
Assert.AreEqual(_afterSetupCounter, 0);
Assert.AreEqual(_setupCounter, 0);
Assert.AreEqual(_cleanupCounter, 0);
Assert.AreEqual(summary?.ValidationErrors.Length, 1);
Assert.IsTrue(summary?.ValidationErrors[0].IsCritical);
Assert.AreEqual(
summary?.ValidationErrors[0].Message,
"Job SelfTestConfigX86, EnvMode.Platform was run as X64 (X86 expected). Fix your test runner options.");
}
/// <summary>Helper for custom configs: configuration that should be used for new performance tests.</summary>
/// <param name="job">The job for the config. Default one will be used if <c>null</c>.</param>
/// <returns>Configuration that should be used for new performance tests.</returns>
public static ManualCompetitionConfig CreateDefaultConfig(IJob job = null)
{
var defaultConfig = BenchmarkDotNet.Configs.DefaultConfig.Instance;
var result = new ManualCompetitionConfig
{
RerunIfLimitsFailed = true,
KeepBenchmarkFiles = defaultConfig.KeepBenchmarkFiles
};
result.Add(defaultConfig.GetColumns().ToArray());
result.Add(defaultConfig.GetValidators().ToArray());
result.Add(defaultConfig.GetAnalysers().ToArray());
result.Add(defaultConfig.GetDiagnosers().ToArray());
result.Set(defaultConfig.GetOrderProvider());
result.Add(job ?? CreateDefaultJob());
return(result);
}
/// <summary>Applies <see cref="IConfigSource"/> attribute modifiers.</summary>
/// <param name="competitionConfig">The competition configuration.</param>
/// <param name="metadataSource">The metadata source.</param>
protected virtual void ApplyConfigSources(ManualCompetitionConfig competitionConfig, ICustomAttributeProvider metadataSource)
{
var config = new ManualConfig();
var add = false;
if (metadataSource != null)
{
foreach (var modifierSource in metadataSource
.GetMetadataAttributes <IConfigSource>()
.Reverse())
{
config = ManualConfig.Union(config, modifierSource.Config);
add = true;
}
}
if (add)
{
competitionConfig.Add(config);
}
}
private static void FixConfigJobs(ManualCompetitionConfig competitionConfig)
{
var jobs = competitionConfig.Jobs;
for (var i = 0; i < jobs.Count; i++)
{
var job = jobs[i];
if (job.Infrastructure.Toolchain == null)
{
var id = job.HasValue(CharacteristicObject.IdCharacteristic) ? job.Id : null;
jobs[i] = job
.With(InProcessToolchain.Instance)
.WithId(id);
}
}
if (competitionConfig.Jobs.Count == 0)
{
competitionConfig.Add(CompetitionConfigFactory.DefaultJob);
}
}
protected virtual ManualCompetitionConfig CreateEmptyConfig(
[CanBeNull] ICustomAttributeProvider metadataSource,
[NotNull] CompetitionFeatures competitionFeatures)
{
var result = new ManualCompetitionConfig(BenchmarkDotNet.Configs.DefaultConfig.Instance);
// DONTTOUCH: competition should not use default
// Jobs, Diagnosers, Exporters and Loggers.
// These omitted intentionally.
result.Jobs.Clear();
result.Diagnosers.Clear();
result.Exporters.Clear();
result.Loggers.Clear();
// TODO: better columns.
// TODO: columns for options.
result.ColumnProviders.Clear();
result.Add(competitionFeatures.TroubleshootingMode ? TroubleshootingModeColumns : DefaultColumns);
var targetAssembly = GetAssembly(metadataSource);
if (targetAssembly != null)
{
if (competitionFeatures.TroubleshootingMode)
{
result.Add(
GetImportantInfoLogger(targetAssembly),
GetDetailedLogger(targetAssembly));
result.Add(Exporters.CsvTimingsExporter.Default);
}
else
{
if (competitionFeatures.ImportantInfoLogger || competitionFeatures.ContinuousIntegrationMode)
{
result.Add(GetImportantInfoLogger(targetAssembly));
}
if (competitionFeatures.DetailedLogger)
{
result.Add(GetDetailedLogger(targetAssembly));
}
}
}
result.Add(DefaultMetrics.ToArray());
return(result);
}
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Add(StatisticColumn.Skewness);
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Add(StatisticColumn.Kurtosis);
public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Add(BaselineScaledColumn.WelchTTestPValue);
