public void WhenTimeoutIsNotSpecifiedTheDefaultValueIsUsed()
{
var mutable = ManualConfig.CreateEmpty();
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Equal(DefaultConfig.Instance.BuildTimeout, final.BuildTimeout);
}
public void WhenArtifactsPathIsNullDefaultValueShouldBeUsed()
{
var mutable = ManualConfig.CreateEmpty();
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Equal(final.ArtifactsPath, DefaultConfig.Instance.ArtifactsPath);
}
public void GenerateWarningWhenExporterDependencyAlreadyExistInConfig()
{
System.Globalization.CultureInfo currentCulture = default;
System.Globalization.CultureInfo currentUICulture = default;
{
var ct = System.Threading.Thread.CurrentThread;
currentCulture = ct.CurrentCulture;
currentUICulture = ct.CurrentUICulture;
ct.CurrentCulture = System.Globalization.CultureInfo.InvariantCulture;
ct.CurrentUICulture = System.Globalization.CultureInfo.InvariantCulture;
}
try
{
var mutable = ManualConfig.CreateEmpty();
mutable.AddExporter(new BenchmarkDotNet.Exporters.Csv.CsvMeasurementsExporter(BenchmarkDotNet.Exporters.Csv.CsvSeparator.Comma));
mutable.AddExporter(RPlotExporter.Default);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Equal(1, final.ConfigAnalysisConclusion.Count);
}
finally
{
var ct = System.Threading.Thread.CurrentThread;
ct.CurrentCulture = currentCulture;
ct.CurrentUICulture = currentUICulture;
}
}
public void WhenSummaryStyleIsNullDefaultValueShouldBeUsed()
{
var mutable = ManualConfig.CreateEmpty();
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Equal(final.SummaryStyle, SummaryStyle.Default);
}
public void WhenOrdererIsNullDefaultValueShouldBeUsed()
{
var mutable = ManualConfig.CreateEmpty();
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Equal(final.Orderer, DefaultOrderer.Instance);
}
public void CustomTimeoutHasPrecedenceOverDefaultTimeout()
{
TimeSpan customTimeout = TimeSpan.FromSeconds(1);
var mutable = ManualConfig.CreateEmpty().WithBuildTimeout(customTimeout);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Equal(customTimeout, final.BuildTimeout);
}
public void JitOptimizationsValidatorIsMandatoryByDefault()
{
var fromEmpty = ImmutableConfigBuilder.Create(ManualConfig.CreateEmpty());
Assert.Contains(JitOptimizationsValidator.DontFailOnError, fromEmpty.GetValidators());
var fromDefault = ImmutableConfigBuilder.Create(DefaultConfig.Instance);
Assert.Contains(JitOptimizationsValidator.FailOnError, fromDefault.GetValidators());
}
public void DuplicateAnalyzersAreExcluded()
{
var mutable = ManualConfig.CreateEmpty();
mutable.AddAnalyser(OutliersAnalyser.Default);
mutable.AddAnalyser(OutliersAnalyser.Default);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Same(OutliersAnalyser.Default, final.GetAnalysers().Single());
}
public void DuplicateDiagnosersAreExcludedBasedOnType()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(DisassemblyDiagnoser.Create(DisassemblyDiagnoserConfig.All));
mutable.Add(DisassemblyDiagnoser.Create(DisassemblyDiagnoserConfig.Asm));
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Single(final.GetDiagnosers());
}
public void DuplicateExportersAreExcluded()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(MarkdownExporter.GitHub);
mutable.Add(MarkdownExporter.GitHub);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Same(MarkdownExporter.GitHub, final.GetExporters().Single());
}
public void DuplicateLoggersAreExcluded()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(ConsoleLogger.Default);
mutable.Add(ConsoleLogger.Default);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Same(ConsoleLogger.Default, final.GetLoggers().Single());
}
public void WhenUserDefinesHardwareCountersWeChooseTheRightDiagnoser()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(HardwareCounter.CacheMisses);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Single(final.GetDiagnosers());
Assert.Single(final.GetDiagnosers().OfType <IHardwareCountersDiagnoser>());
}
public void DuplicateHardwareCountersAreExcluded()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(HardwareCounter.CacheMisses);
mutable.Add(HardwareCounter.CacheMisses);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Equal(HardwareCounter.CacheMisses, final.GetHardwareCounters().Single());
}
public void DuplicateColumnProvidersAreExcluded()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(DefaultColumnProviders.Job);
mutable.Add(DefaultColumnProviders.Job);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Same(DefaultColumnProviders.Job, final.GetColumnProviders().Single());
}
[Fact] // See https://github.com/dotnet/BenchmarkDotNet/issues/172
public void MissingExporterDependencyIsAddedWhenNeeded()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(TestExporter.Default);
var exporters = ImmutableConfigBuilder.Create(mutable).GetExporters().ToArray();
Assert.Equal(2, exporters.Length);
Assert.Equal(new IExporter[] { TestExporterDependency.Default, TestExporter.Default }, exporters);
}
public void DuplicateValidatorsAreExcludedBasedOnTreatsWarningsAsErrorsProperty()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(JitOptimizationsValidator.DontFailOnError);
mutable.Add(JitOptimizationsValidator.FailOnError);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Same(JitOptimizationsValidator.FailOnError, final.GetValidators().OfType <JitOptimizationsValidator>().Single());
}
public void JitOptimizationsValidatorIsMandatoryByDefault()
{
var fromEmpty = ImmutableConfigBuilder.Create(ManualConfig.CreateEmpty());
Assert.Contains(JitOptimizationsValidator.DontFailOnError, fromEmpty.GetValidators());
#if !DEBUG
// DefaultConfig.Instance doesn't include JitOptimizationsValidator.FailOnError in the DEBUG mode
var fromDefault = ImmutableConfigBuilder.Create(DefaultConfig.Instance);
Assert.Contains(JitOptimizationsValidator.FailOnError, fromDefault.GetValidators());
#endif
}
public void WhenUserDefinesHardwareCountersAndUsesDisassemblyDiagnoserWeAddInstructionPointerExporter()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(HardwareCounter.CacheMisses);
mutable.Add(DisassemblyDiagnoser.Create(DisassemblyDiagnoserConfig.All));
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Single(final.GetDiagnosers().OfType <IHardwareCountersDiagnoser>());
Assert.Single(final.GetDiagnosers().OfType <IDisassemblyDiagnoser>());
Assert.Single(final.GetExporters().OfType <InstructionPointerExporter>());
}
private static ImmutableConfig GetFullTypeConfig(Type type, IConfig config)
{
config = config ?? DefaultConfig.Instance;
var typeAttributes = type.GetCustomAttributes(true).OfType <IConfigSource>();
var assemblyAttributes = type.Assembly.GetCustomAttributes().OfType <IConfigSource>();
foreach (var configFromAttribute in typeAttributes.Concat(assemblyAttributes))
{
config = ManualConfig.Union(config, configFromAttribute.Config);
}
return(ImmutableConfigBuilder.Create(config));
}
public void JitOptimizationsValidatorIsMandatoryCanBeDisabledOnDemand()
{
var disabled = ImmutableConfigBuilder.Create(ManualConfig.CreateEmpty().With(ConfigOptions.DisableOptimizationsValidator));
Assert.DoesNotContain(JitOptimizationsValidator.FailOnError, disabled.GetValidators());
Assert.DoesNotContain(JitOptimizationsValidator.DontFailOnError, disabled.GetValidators());
var enabledThenDisabled = ImmutableConfigBuilder.Create(ManualConfig.CreateEmpty()
.With(JitOptimizationsValidator.FailOnError) // we enable it first (to mimic few configs merge)
.With(ConfigOptions.DisableOptimizationsValidator)); // then disable
Assert.DoesNotContain(JitOptimizationsValidator.FailOnError, enabledThenDisabled.GetValidators());
Assert.DoesNotContain(JitOptimizationsValidator.DontFailOnError, enabledThenDisabled.GetValidators());
}
private static Summary CreateMockSummary(bool printUnitsInContent, bool printUnitsInHeader, TimeUnit timeUnit, double metricValue)
{
var summaryStyle = new SummaryStyle(TestCultureInfo.Instance, printUnitsInHeader, null, timeUnit, printUnitsInContent);
var config = new ManualConfig().WithSummaryStyle(summaryStyle);
var benchmarkCase = new BenchmarkCase(
new Descriptor(null, null),
Job.Dry,
new ParameterInstances(ImmutableArray <ParameterInstance> .Empty),
ImmutableConfigBuilder.Create(config));
var metric = new Metric(LocalMetricDescriptor.TimeInstance, metricValue);
var benchmarkReport = new BenchmarkReport(true, benchmarkCase, null, null, null, new List <Metric>()
{
metric
});
return(new Summary("", new[] { benchmarkReport }.ToImmutableArray(), HostEnvironmentInfo.GetCurrent(),
"", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray <ValidationError> .Empty, ImmutableArray <IColumnHidingRule> .Empty));
}
public void WhenTwoCustomTimeoutsAreProvidedTheLongerOneIsUsed(bool direction)
{
var oneSecond = ManualConfig.CreateEmpty().WithBuildTimeout(TimeSpan.FromSeconds(1));
var twoSeconds = ManualConfig.CreateEmpty().WithBuildTimeout(TimeSpan.FromSeconds(2));
if (direction)
{
oneSecond.Add(twoSeconds);
}
else
{
twoSeconds.Add(oneSecond);
}
var final = ImmutableConfigBuilder.Create(direction ? oneSecond : twoSeconds);
Assert.Equal(TimeSpan.FromSeconds(2), final.BuildTimeout);
}
private static ImmutableConfig GetFullMethodConfig(MethodInfo method, ImmutableConfig typeConfig)
{
var methodAttributes = method.GetCustomAttributes(true).OfType <IConfigSource>();
if (!methodAttributes.Any()) // the most common case
{
return(typeConfig);
}
var config = ManualConfig.Create(typeConfig);
foreach (var configFromAttribute in methodAttributes)
{
config = ManualConfig.Union(config, configFromAttribute.Config);
}
return(ImmutableConfigBuilder.Create(config));
}
public static ImmutableConfig GetFullConfig(Type type, IConfig config)
{
config = config ?? DefaultConfig.Instance;
if (type != null)
{
var typeAttributes = type.GetTypeInfo().GetCustomAttributes(true).OfType <IConfigSource>();
var assemblyAttributes = type.GetTypeInfo().Assembly.GetCustomAttributes().OfType <IConfigSource>();
var allAttributes = typeAttributes.Concat(assemblyAttributes);
var configs = allAttributes.Select(attribute => attribute.Config)
.OrderBy(c => c.GetJobs().Count(job => job.Meta.IsMutator)); // configs with mutators must be the ones applied at the end
foreach (var configFromAttribute in configs)
{
config = ManualConfig.Union(config, configFromAttribute);
}
}
return(ImmutableConfigBuilder.Create(config));
}
public void BaseLineValidatorIsMandatory()
{
var fromEmpty = ImmutableConfigBuilder.Create(ManualConfig.CreateEmpty());
Assert.Contains(BaselineValidator.FailOnError, fromEmpty.GetValidators());
}
