/// <summary> /// Initializes a new instance of the <see cref="QuickRunConfigAttribute"/> class. /// </summary> public QuickRunConfigAttribute() { Config = (ManualConfig)ManualConfig.CreateEmpty() .With(Job.Default.WithLaunchCount(1) // benchmark process will be launched only once .WithIterationTime(new TimeInterval(100, TimeUnit.Millisecond)) // 100ms per iteration .WithWarmupCount(3) // 3 warmup iteration .WithTargetCount(3)); // 3 target iteration }
public SingleRunJobAttribute() { Config = ManualConfig.CreateEmpty().With( new Job() .WithWarmupCount(1) .WithIterationCount(1) .With(RunStrategy.Monitoring)); }
public BenchmarkJobAttribute(bool singleRun = false) { var job = new Job().ConfigureJob(singleRun); Config = ManualConfig.CreateEmpty() .With(job.With(CsProjClassicNetToolchain.Net461)) .With(job.With(CsProjCoreToolchain.NetCoreApp21)); }
private static ManualConfig CreateConfigFromJobs(params Job[] jobs) { var config = ManualConfig.CreateEmpty(); config.Add(jobs); return(config); }
public void CustomAffinityCanBeSet() { var config = ManualConfig.CreateEmpty() .With(Job.Dry.WithAffinity(CustomAffinity.Value)) .With(DefaultColumnProviders.Instance) .With(new OutputLogger(Output)); CanExecute <CustomAffinity>(config); }
private IConfig CreateConfig(IToolchain toolchain) { return(ManualConfig.CreateEmpty() .With(Job.ShortRun.WithGcForce(false).With(toolchain)) .With(DefaultConfig.Instance.GetLoggers().ToArray()) .With(DefaultColumnProviders.Instance) .With(MemoryDiagnoser.Default) .With(new OutputLogger(output))); }
public JemBenchmarkJobAttribute( RunStrategy RunStrategy = RunStrategy.Throughput, int TargetCount = DefaultValue, int InvocationCount = DefaultValue, int WarmupCount = DefaultValue ) { Job job = new Job() .WithGcAllowVeryLargeObjects(true) .WithId("JemBenchmark"); job.Env.Platform = Platform.X64; job.Env.Runtime = Runtime.Core; job.Env.Jit = Jit.RyuJit; if (WarmupCountOverride.HasValue) { job.Run.WarmupCount = WarmupCountOverride.Value; } else if (WarmupCount != DefaultValue) { job.Run.WarmupCount = WarmupCount; } if (TargetCountOverride.HasValue) { job.Run.TargetCount = TargetCountOverride.Value; } else if (TargetCount != DefaultValue) { job.Run.TargetCount = TargetCount; } if (InvocationCountOverride.HasValue) { job.Run.InvocationCount = InvocationCountOverride.Value; } else if (InvocationCount != DefaultValue) { job.Run.InvocationCount = InvocationCount; } if (ColdStartOverride.HasValue) { job.Run.RunStrategy = ColdStartOverride.Value ? RunStrategy.ColdStart : RunStrategy.Throughput; } else { job.Run.RunStrategy = RunStrategy; } job.Infrastructure.Toolchain = new InProcessToolchain(TimeSpan.FromMinutes(TimeoutInMinutes), BenchmarkActionCodegen.ReflectionEmit, true); Config = ManualConfig.CreateEmpty().With(job); }
public void CanAllowToCreateVeryLargeObjectsFor64Bit() { var config = ManualConfig.CreateEmpty() .With(Job.Dry.With(Platform.X64).With(new GarbageCollection { AllowVeryLargeObjects = true })); CanExecute <CreateVeryLargeObjects>(config); }
public void CanAvoidForcingGarbageCollections() { var config = ManualConfig.CreateEmpty() .With(Job.Dry.With(new GarbageCollection { Force = false })); CanExecute <AvoidForcingGarbageCollection>(config); }
public void CanDisableConcurrentGcMode() { var config = ManualConfig.CreateEmpty() .With(Job.Dry.With(new GarbageCollection { Concurrent = false })); CanExecute <ConcurrentModeDisabled>(config); }
public void CanDisableServerGcMode() { var config = ManualConfig.CreateEmpty() .With(Job.Dry.With(new GarbageCollection { Server = false })); CanExecute <WorkstationGcOnly>(config); }
public void CanEnableServerGcMode() { var config = ManualConfig.CreateEmpty() .With(Job.Dry.With(new GarbageCollection { Server = true })); CanExecute <ServerModeEnabled>(config); }
public void CustomTimeoutHasPrecedenceOverDefaultTimeout() { TimeSpan customTimeout = TimeSpan.FromSeconds(1); var mutable = ManualConfig.CreateEmpty().WithBuildTimeout(customTimeout); var final = ImmutableConfigBuilder.Create(mutable); Assert.Equal(customTimeout, final.BuildTimeout); }
private void SourceExceptionMessageIsDisplayed <TBenchmark>(Job job) { var logger = new AccumulationLogger(); var config = ManualConfig.CreateEmpty().With(job).With(logger); CanExecute <TBenchmark>(config, fullValidation: false); // we don't validate here because the report is expected to have no results Assert.Contains(BenchmarkExceptionMessage, logger.GetLog()); }
public static void Run() { BenchmarkRunner.Run <IntroCustomMonoFluentConfig>(ManualConfig .CreateEmpty() .With(Job.ShortRun.With(new MonoRuntime( "Mono x64", @"C:\Program Files\Mono\bin\mono.exe"))) .With(Job.ShortRun.With(new MonoRuntime( "Mono x86", @"C:\Program Files (x86)\Mono\bin\mono.exe")))); }
private IConfig CreateConfig(IDiagnoser diagnoser) { return(ManualConfig.CreateEmpty() .With(Job.ShortRun.WithGcForce(false)) .With(DefaultConfig.Instance.GetLoggers().ToArray()) .With(DefaultColumnProviders.Instance) .With(diagnoser) .With(new OutputLogger(output))); }
public void BenchmarkRunnerShouldNotFailOnCriticalValidationErrors() { BenchmarkRunner .Run <ValidatorsTest>( ManualConfig .CreateEmpty() .With(new FailingValidator()) .With(ConsoleLogger.Default) // so we get an output in the TestRunner log .With(AllKnownExportersThatSupportExportToLog)); }
private IConfig CreateConfig(Jit jit, Platform platform, Runtime runtime, IDiagnoser disassemblyDiagnoser, RunStrategy runStrategy) => ManualConfig.CreateEmpty() .AddJob(Job.Dry.WithJit(jit) .WithPlatform(platform) .WithRuntime(runtime) .WithStrategy(runStrategy)) .AddLogger(DefaultConfig.Instance.GetLoggers().ToArray()) .AddColumnProvider(DefaultColumnProviders.Instance) .AddDiagnoser(disassemblyDiagnoser) .AddLogger(new OutputLogger(Output));
public void JitOptimizationsValidatorIsMandatoryByDefault() { var fromEmpty = ImmutableConfigBuilder.Create(ManualConfig.CreateEmpty()); Assert.Contains(JitOptimizationsValidator.DontFailOnError, fromEmpty.GetValidators()); var fromDefault = ImmutableConfigBuilder.Create(DefaultConfig.Instance); Assert.Contains(JitOptimizationsValidator.FailOnError, fromDefault.GetValidators()); }
public void CanBenchmarkLocalCoreRtUsingRyuJit() { var config = ManualConfig.CreateEmpty() .With(Job.DryCoreRT.With( CoreRtToolchain.CreateBuilder() .UseCoreRtLocal(IlcPath) .ToToolchain())); CanExecute <CoreRtBenchmark>(config); }
private static IConfig GetConfig() { #if NET5_0 || NETCOREAPP5_0 return(ManualConfig.CreateEmpty() .AddJob(Job.Default .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings ( // not using "net5.0", a workaround for https://github.com/dotnet/BenchmarkDotNet/pull/1479 targetFrameworkMoniker: "netcoreapp5.0", runtimeFrameworkVersion: default,
public static IConfig Create( DirectoryInfo artifactsPath, ImmutableHashSet <string> mandatoryCategories, int?partitionCount = null, int?partitionIndex = null, List <string> exclusionFilterValue = null, List <string> categoryExclusionFilterValue = null, Job job = null, bool getDiffableDisasm = false) { if (job is null) { job = Job.Default .WithWarmupCount(1) // 1 warmup is enough for our purpose .WithIterationTime(TimeInterval.FromMilliseconds(250)) // the default is 0.5s per iteration, which is slighlty too much for us .WithMinIterationCount(15) .WithMaxIterationCount(20) // we don't want to run more that 20 iterations .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows // See https://github.com/dotnet/roslyn/issues/42393 job = job.WithArguments(new Argument[] { new MsBuildArgument("/p:DebugType=portable"), new MsBuildArgument("-bl:benchmarkdotnet.binlog") }); } var config = ManualConfig.CreateEmpty() .WithBuildTimeout(TimeSpan.FromMinutes(10)) // for slow machines .AddLogger(ConsoleLogger.Default) // log output to console .AddValidator(DefaultConfig.Instance.GetValidators().ToArray()) // copy default validators .AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) // copy default analysers .AddExporter(MarkdownExporter.GitHub) // export to GitHub markdown .AddColumnProvider(DefaultColumnProviders.Instance) // display default columns (method name, args etc) .AddJob(job.AsDefault()) // tell BDN that this are our default settings .WithArtifactsPath(artifactsPath.FullName) .AddDiagnoser(MemoryDiagnoser.Default) // MemoryDiagnoser is enabled by default .AddFilter(new PartitionFilter(partitionCount, partitionIndex)) .AddFilter(new ExclusionFilter(exclusionFilterValue)) .AddFilter(new CategoryExclusionFilter(categoryExclusionFilterValue)) .AddExporter(JsonExporter.Full) // make sure we export to Json .AddColumn(StatisticColumn.Median, StatisticColumn.Min, StatisticColumn.Max) .AddValidator(TooManyTestCasesValidator.FailOnError) .AddValidator(new UniqueArgumentsValidator()) // don't allow for duplicated arguments #404 .AddValidator(new MandatoryCategoryValidator(mandatoryCategories)) .WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(36)); // the default is 20 and trims too aggressively some benchmark results if (Reporter.CreateReporter().InLab) { config = config.AddExporter(new PerfLabExporter()); } if (getDiffableDisasm) { config = config.AddDiagnoser(CreateDisassembler()); } return(config); }
public MyConfigSourceAttribute(params Jit[] jits) { var jobs = jits .Select(jit => new Job(Job.Dry) { Env = { Jit = jit, Platform = Platform.X64 } }) .ToArray(); Config = ManualConfig.CreateEmpty().With(jobs); }
public void CoreRtIsSupported() { var config = ManualConfig.CreateEmpty() .With(Job.Dry .With(Runtime.CoreRT) .With(CoreRtToolchain.CreateBuilder() .UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "1.0.0-alpha-26414-01") // we test against specific version to keep this test stable .ToToolchain())); CanExecute <CoreRtBenchmark>(config); }
public void CustomEnginesAreSupported() { var config = ManualConfig.CreateEmpty() .With(Job.Dry.With(new CustomFactory())); var summary = CanExecute <SimpleBenchmark>(config, fullValidation: false); AssertMessageGotDisplayed(summary, SetupMessage); AssertMessageGotDisplayed(summary, EngineRunMessage); AssertMessageGotDisplayed(summary, CleanupMessage); }
public void CanBenchmarkLocalCoreRtUsingCppCodeGen() { var config = ManualConfig.CreateEmpty() .With(Job.DryCoreRT.With( CoreRtToolchain.CreateBuilder() .UseCoreRtLocal(IlcPath) .UseCppCodeGenerator() // https://github.com/dotnet/corert/blob/7f902d4d8b1c3280e60f5e06c71951a60da173fb/Documentation/how-to-build-and-run-ilcompiler-in-console-shell-prompt.md#using-cpp-code-generator .ToToolchain())); CanExecute <CoreRtBenchmark>(config); }
private static IConfig GetConfig() { #if DEBUG return(new DebugInProcessConfig()); #else return(ManualConfig.CreateEmpty() .AddJob(Job.Default .WithToolchain(CsProjCoreToolchain.From(NetCoreAppSettings.NetCoreApp60)) .AsDefault())); #endif }
public void DuplicateDiagnosersAreExcludedBasedOnType() { var mutable = ManualConfig.CreateEmpty(); mutable.Add(DisassemblyDiagnoser.Create(DisassemblyDiagnoserConfig.All)); mutable.Add(DisassemblyDiagnoser.Create(DisassemblyDiagnoserConfig.Asm)); var final = ImmutableConfigBuilder.Create(mutable); Assert.Single(final.GetDiagnosers()); }
public void WhenUserDefinesHardwareCountersWeChooseTheRightDiagnoser() { var mutable = ManualConfig.CreateEmpty(); mutable.Add(HardwareCounter.CacheMisses); var final = ImmutableConfigBuilder.Create(mutable); Assert.Single(final.GetDiagnosers()); Assert.Single(final.GetDiagnosers().OfType <IHardwareCountersDiagnoser>()); }
public void DuplicateHardwareCountersAreExcluded() { var mutable = ManualConfig.CreateEmpty(); mutable.Add(HardwareCounter.CacheMisses); mutable.Add(HardwareCounter.CacheMisses); var final = ImmutableConfigBuilder.Create(mutable); Assert.Equal(HardwareCounter.CacheMisses, final.GetHardwareCounters().Single()); }