/// <summary>
/// Initializes a new instance of the <see cref="QuickRunConfigAttribute"/> class.
/// </summary>
public QuickRunConfigAttribute()
{
Config = (ManualConfig)ManualConfig.CreateEmpty()
.With(Job.Default.WithLaunchCount(1) // benchmark process will be launched only once
.WithIterationTime(new TimeInterval(100, TimeUnit.Millisecond)) // 100ms per iteration
.WithWarmupCount(3) // 3 warmup iteration
.WithTargetCount(3)); // 3 target iteration
}
public SingleRunJobAttribute()
{
Config = ManualConfig.CreateEmpty().With(
new Job()
.WithWarmupCount(1)
.WithIterationCount(1)
.With(RunStrategy.Monitoring));
}
public BenchmarkJobAttribute(bool singleRun = false)
{
var job = new Job().ConfigureJob(singleRun);
Config = ManualConfig.CreateEmpty()
.With(job.With(CsProjClassicNetToolchain.Net461))
.With(job.With(CsProjCoreToolchain.NetCoreApp21));
}
private static ManualConfig CreateConfigFromJobs(params Job[] jobs)
{
var config = ManualConfig.CreateEmpty();
config.Add(jobs);
return(config);
}
public void CustomAffinityCanBeSet()
{
var config = ManualConfig.CreateEmpty()
.With(Job.Dry.WithAffinity(CustomAffinity.Value))
.With(DefaultColumnProviders.Instance)
.With(new OutputLogger(Output));
CanExecute <CustomAffinity>(config);
}
private IConfig CreateConfig(IToolchain toolchain)
{
return(ManualConfig.CreateEmpty()
.With(Job.ShortRun.WithGcForce(false).With(toolchain))
.With(DefaultConfig.Instance.GetLoggers().ToArray())
.With(DefaultColumnProviders.Instance)
.With(MemoryDiagnoser.Default)
.With(new OutputLogger(output)));
}
public JemBenchmarkJobAttribute(
RunStrategy RunStrategy = RunStrategy.Throughput,
int TargetCount = DefaultValue,
int InvocationCount = DefaultValue,
int WarmupCount = DefaultValue
)
{
Job job = new Job()
.WithGcAllowVeryLargeObjects(true)
.WithId("JemBenchmark");
job.Env.Platform = Platform.X64;
job.Env.Runtime = Runtime.Core;
job.Env.Jit = Jit.RyuJit;
if (WarmupCountOverride.HasValue)
{
job.Run.WarmupCount = WarmupCountOverride.Value;
}
else if (WarmupCount != DefaultValue)
{
job.Run.WarmupCount = WarmupCount;
}
if (TargetCountOverride.HasValue)
{
job.Run.TargetCount = TargetCountOverride.Value;
}
else if (TargetCount != DefaultValue)
{
job.Run.TargetCount = TargetCount;
}
if (InvocationCountOverride.HasValue)
{
job.Run.InvocationCount = InvocationCountOverride.Value;
}
else if (InvocationCount != DefaultValue)
{
job.Run.InvocationCount = InvocationCount;
}
if (ColdStartOverride.HasValue)
{
job.Run.RunStrategy = ColdStartOverride.Value ? RunStrategy.ColdStart : RunStrategy.Throughput;
}
else
{
job.Run.RunStrategy = RunStrategy;
}
job.Infrastructure.Toolchain = new InProcessToolchain(TimeSpan.FromMinutes(TimeoutInMinutes), BenchmarkActionCodegen.ReflectionEmit, true);
Config = ManualConfig.CreateEmpty().With(job);
}
public void CanAllowToCreateVeryLargeObjectsFor64Bit()
{
var config = ManualConfig.CreateEmpty()
.With(Job.Dry.With(Platform.X64).With(new GarbageCollection {
AllowVeryLargeObjects = true
}));
CanExecute <CreateVeryLargeObjects>(config);
}
public void CanAvoidForcingGarbageCollections()
{
var config = ManualConfig.CreateEmpty()
.With(Job.Dry.With(new GarbageCollection {
Force = false
}));
CanExecute <AvoidForcingGarbageCollection>(config);
}
public void CanDisableConcurrentGcMode()
{
var config = ManualConfig.CreateEmpty()
.With(Job.Dry.With(new GarbageCollection {
Concurrent = false
}));
CanExecute <ConcurrentModeDisabled>(config);
}
public void CanDisableServerGcMode()
{
var config = ManualConfig.CreateEmpty()
.With(Job.Dry.With(new GarbageCollection {
Server = false
}));
CanExecute <WorkstationGcOnly>(config);
}
public void CanEnableServerGcMode()
{
var config = ManualConfig.CreateEmpty()
.With(Job.Dry.With(new GarbageCollection {
Server = true
}));
CanExecute <ServerModeEnabled>(config);
}
public void CustomTimeoutHasPrecedenceOverDefaultTimeout()
{
TimeSpan customTimeout = TimeSpan.FromSeconds(1);
var mutable = ManualConfig.CreateEmpty().WithBuildTimeout(customTimeout);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Equal(customTimeout, final.BuildTimeout);
}
private void SourceExceptionMessageIsDisplayed <TBenchmark>(Job job)
{
var logger = new AccumulationLogger();
var config = ManualConfig.CreateEmpty().With(job).With(logger);
CanExecute <TBenchmark>(config, fullValidation: false); // we don't validate here because the report is expected to have no results
Assert.Contains(BenchmarkExceptionMessage, logger.GetLog());
}
public static void Run()
{
BenchmarkRunner.Run <IntroCustomMonoFluentConfig>(ManualConfig
.CreateEmpty()
.With(Job.ShortRun.With(new MonoRuntime(
"Mono x64", @"C:\Program Files\Mono\bin\mono.exe")))
.With(Job.ShortRun.With(new MonoRuntime(
"Mono x86", @"C:\Program Files (x86)\Mono\bin\mono.exe"))));
}
private IConfig CreateConfig(IDiagnoser diagnoser)
{
return(ManualConfig.CreateEmpty()
.With(Job.ShortRun.WithGcForce(false))
.With(DefaultConfig.Instance.GetLoggers().ToArray())
.With(DefaultColumnProviders.Instance)
.With(diagnoser)
.With(new OutputLogger(output)));
}
public void BenchmarkRunnerShouldNotFailOnCriticalValidationErrors()
{
BenchmarkRunner
.Run <ValidatorsTest>(
ManualConfig
.CreateEmpty()
.With(new FailingValidator())
.With(ConsoleLogger.Default) // so we get an output in the TestRunner log
.With(AllKnownExportersThatSupportExportToLog));
}
private IConfig CreateConfig(Jit jit, Platform platform, Runtime runtime, IDiagnoser disassemblyDiagnoser, RunStrategy runStrategy)
=> ManualConfig.CreateEmpty()
.AddJob(Job.Dry.WithJit(jit)
.WithPlatform(platform)
.WithRuntime(runtime)
.WithStrategy(runStrategy))
.AddLogger(DefaultConfig.Instance.GetLoggers().ToArray())
.AddColumnProvider(DefaultColumnProviders.Instance)
.AddDiagnoser(disassemblyDiagnoser)
.AddLogger(new OutputLogger(Output));
public void JitOptimizationsValidatorIsMandatoryByDefault()
{
var fromEmpty = ImmutableConfigBuilder.Create(ManualConfig.CreateEmpty());
Assert.Contains(JitOptimizationsValidator.DontFailOnError, fromEmpty.GetValidators());
var fromDefault = ImmutableConfigBuilder.Create(DefaultConfig.Instance);
Assert.Contains(JitOptimizationsValidator.FailOnError, fromDefault.GetValidators());
}
public void CanBenchmarkLocalCoreRtUsingRyuJit()
{
var config = ManualConfig.CreateEmpty()
.With(Job.DryCoreRT.With(
CoreRtToolchain.CreateBuilder()
.UseCoreRtLocal(IlcPath)
.ToToolchain()));
CanExecute <CoreRtBenchmark>(config);
}
private static IConfig GetConfig()
{
#if NET5_0 || NETCOREAPP5_0
return(ManualConfig.CreateEmpty()
.AddJob(Job.Default
.WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings
(
// not using "net5.0", a workaround for https://github.com/dotnet/BenchmarkDotNet/pull/1479
targetFrameworkMoniker: "netcoreapp5.0",
runtimeFrameworkVersion: default,
public static IConfig Create(
DirectoryInfo artifactsPath,
ImmutableHashSet <string> mandatoryCategories,
int?partitionCount = null,
int?partitionIndex = null,
List <string> exclusionFilterValue = null,
List <string> categoryExclusionFilterValue = null,
Job job = null,
bool getDiffableDisasm = false)
{
if (job is null)
{
job = Job.Default
.WithWarmupCount(1) // 1 warmup is enough for our purpose
.WithIterationTime(TimeInterval.FromMilliseconds(250)) // the default is 0.5s per iteration, which is slighlty too much for us
.WithMinIterationCount(15)
.WithMaxIterationCount(20) // we don't want to run more that 20 iterations
.DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows
// See https://github.com/dotnet/roslyn/issues/42393
job = job.WithArguments(new Argument[] { new MsBuildArgument("/p:DebugType=portable"), new MsBuildArgument("-bl:benchmarkdotnet.binlog") });
}
var config = ManualConfig.CreateEmpty()
.WithBuildTimeout(TimeSpan.FromMinutes(10)) // for slow machines
.AddLogger(ConsoleLogger.Default) // log output to console
.AddValidator(DefaultConfig.Instance.GetValidators().ToArray()) // copy default validators
.AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) // copy default analysers
.AddExporter(MarkdownExporter.GitHub) // export to GitHub markdown
.AddColumnProvider(DefaultColumnProviders.Instance) // display default columns (method name, args etc)
.AddJob(job.AsDefault()) // tell BDN that this are our default settings
.WithArtifactsPath(artifactsPath.FullName)
.AddDiagnoser(MemoryDiagnoser.Default) // MemoryDiagnoser is enabled by default
.AddFilter(new PartitionFilter(partitionCount, partitionIndex))
.AddFilter(new ExclusionFilter(exclusionFilterValue))
.AddFilter(new CategoryExclusionFilter(categoryExclusionFilterValue))
.AddExporter(JsonExporter.Full) // make sure we export to Json
.AddColumn(StatisticColumn.Median, StatisticColumn.Min, StatisticColumn.Max)
.AddValidator(TooManyTestCasesValidator.FailOnError)
.AddValidator(new UniqueArgumentsValidator()) // don't allow for duplicated arguments #404
.AddValidator(new MandatoryCategoryValidator(mandatoryCategories))
.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(36)); // the default is 20 and trims too aggressively some benchmark results
if (Reporter.CreateReporter().InLab)
{
config = config.AddExporter(new PerfLabExporter());
}
if (getDiffableDisasm)
{
config = config.AddDiagnoser(CreateDisassembler());
}
return(config);
}
public MyConfigSourceAttribute(params Jit[] jits)
{
var jobs = jits
.Select(jit => new Job(Job.Dry)
{
Env = { Jit = jit, Platform = Platform.X64 }
})
.ToArray();
Config = ManualConfig.CreateEmpty().With(jobs);
}
public void CoreRtIsSupported()
{
var config = ManualConfig.CreateEmpty()
.With(Job.Dry
.With(Runtime.CoreRT)
.With(CoreRtToolchain.CreateBuilder()
.UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "1.0.0-alpha-26414-01") // we test against specific version to keep this test stable
.ToToolchain()));
CanExecute <CoreRtBenchmark>(config);
}
public void CustomEnginesAreSupported()
{
var config = ManualConfig.CreateEmpty()
.With(Job.Dry.With(new CustomFactory()));
var summary = CanExecute <SimpleBenchmark>(config, fullValidation: false);
AssertMessageGotDisplayed(summary, SetupMessage);
AssertMessageGotDisplayed(summary, EngineRunMessage);
AssertMessageGotDisplayed(summary, CleanupMessage);
}
public void CanBenchmarkLocalCoreRtUsingCppCodeGen()
{
var config = ManualConfig.CreateEmpty()
.With(Job.DryCoreRT.With(
CoreRtToolchain.CreateBuilder()
.UseCoreRtLocal(IlcPath)
.UseCppCodeGenerator() // https://github.com/dotnet/corert/blob/7f902d4d8b1c3280e60f5e06c71951a60da173fb/Documentation/how-to-build-and-run-ilcompiler-in-console-shell-prompt.md#using-cpp-code-generator
.ToToolchain()));
CanExecute <CoreRtBenchmark>(config);
}
private static IConfig GetConfig()
{
#if DEBUG
return(new DebugInProcessConfig());
#else
return(ManualConfig.CreateEmpty()
.AddJob(Job.Default
.WithToolchain(CsProjCoreToolchain.From(NetCoreAppSettings.NetCoreApp60))
.AsDefault()));
#endif
}
public void DuplicateDiagnosersAreExcludedBasedOnType()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(DisassemblyDiagnoser.Create(DisassemblyDiagnoserConfig.All));
mutable.Add(DisassemblyDiagnoser.Create(DisassemblyDiagnoserConfig.Asm));
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Single(final.GetDiagnosers());
}
public void WhenUserDefinesHardwareCountersWeChooseTheRightDiagnoser()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(HardwareCounter.CacheMisses);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Single(final.GetDiagnosers());
Assert.Single(final.GetDiagnosers().OfType <IHardwareCountersDiagnoser>());
}
public void DuplicateHardwareCountersAreExcluded()
{
var mutable = ManualConfig.CreateEmpty();
mutable.Add(HardwareCounter.CacheMisses);
mutable.Add(HardwareCounter.CacheMisses);
var final = ImmutableConfigBuilder.Create(mutable);
Assert.Equal(HardwareCounter.CacheMisses, final.GetHardwareCounters().Single());
}