private static void RunCore(Type benchmarkType, CompetitionState competitionState)
{
var logger = competitionState.Logger;
var runOptions = competitionState.Options.RunOptions;
Code.InRange(
runOptions.MaxRunsAllowed,
CompetitionRunMode.MaxRunsAllowedCharacteristic.FullId,
0, MaxRunLimit);
while (competitionState.RunsLeft > 0)
{
competitionState.PrepareForRun();
var run = competitionState.RunNumber;
var runsExpected = competitionState.RunNumber + competitionState.RunsLeft;
var runMessage = competitionState.RunLimitExceeded
? $"Run {run}, total runs (expected): {runsExpected} (rerun limit exceeded, last run)"
: $"Run {run}, total runs (expected): {runsExpected}";
using (BeginLogImportant(competitionState.Config))
{
logger.WriteSeparatorLine(runMessage);
}
// TODO: toolchainProvider to base (???).
Func <Job, IToolchain> toolchainProvider = j => j.Infrastructure?.Toolchain ?? InProcessToolchain.Instance;
// Running the benchmark
var summary = BenchmarkRunnerCore.Run(
BenchmarkConverter.TypeToBenchmarks(benchmarkType, competitionState.Config),
competitionState.Config,
toolchainProvider);
competitionState.RunCompleted(summary);
// Dump messages if analysis was not run and there is a validation analyser.
if (summary.HasCriticalValidationErrors)
{
var validationAnalyser = competitionState.Config
.GetAnalysers()
.OfType <ValidatorMessagesAnalyser>()
.FirstOrDefault();
validationAnalyser?.Analyse(summary);
}
if (competitionState.HasCriticalErrorsInRun)
{
competitionState.Logger.WriteVerboseHint("Breaking competition execution. High severity error occured.");
break;
}
if (competitionState.RunLimitExceeded)
{
break;
}
if (competitionState.RunsLeft > 0)
{
competitionState.Logger.WriteVerboseHint($"Rerun requested. Runs left: {competitionState.RunsLeft}.");
}
}
if (competitionState.RunLimitExceeded && competitionState.RunsLeft > 0)
{
competitionState.WriteMessage(
MessageSource.Runner, MessageSeverity.TestError,
$"The benchmark run limit ({runOptions.MaxRunsAllowed} runs(s)) exceeded, check log for details.");
}
else if (competitionState.RunNumber > 1)
{
competitionState.WriteMessage(
MessageSource.Runner, MessageSeverity.Warning,
$"The benchmark was run {competitionState.RunNumber} time(s), check log for details.");
}
}
public void FailingBenchmarksAreDiscovered()
{
var validationErrors = ExecutionValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(FailingBenchmark)));
Assert.NotEmpty(validationErrors);
}
public void Benchmark_Class_Methods_Must_Be_Non_Static(Type type, bool hasErrors)
{
var validationErrors = CompilationValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(type));
Assert.Equal(hasErrors, validationErrors.Any());
}
public void NonFailingSetupsAreOmmited()
{
var validationErrors = ExecutionValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(SetupThatRequiresParamsToBeSetFirst)));
Assert.Empty(validationErrors);
}
public void FieldsWithoutParamsValuesAreDiscovered()
{
Assert.Empty(BenchmarkConverter.TypeToBenchmarks(typeof(FieldsWithoutParamsValues)));
}
public static void Main()
{
//_ = BenchmarkRunner.Run<ConnectionPoolBenchmark>();
//_ = BenchmarkRunner.Run<ChannelPoolBenchmark>();
//_ = BenchmarkRunner.Run<UtilsBenchmark>();
//_ = BenchmarkRunner.Run<EncryptBenchmark>();
//_ = BenchmarkRunner.Run<BouncyEncryptBenchmark>();
//BenchmarkRunner.Run(
// new[]
// {
// BenchmarkConverter.TypeToBenchmarks(typeof(BouncyEncryptBenchmark))
// });
//BenchmarkRunner.Run(
// new[]
// {
// BenchmarkConverter.TypeToBenchmarks(typeof(GzipBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(RecyclableGzipBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(BrotliBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(RecyclableBrotliBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(DeflateBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(RecyclableDeflateBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(LZ4PickleBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(LZ4StreamBenchmark))
// });
//BenchmarkRunner.Run(
// new[]
// {
// BenchmarkConverter.TypeToBenchmarks(typeof(DataTransformerBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(BouncyDataTransformBenchmark)),
// });
//BenchmarkRunner.Run(
// new[]
// {
// BenchmarkConverter.TypeToBenchmarks(typeof(MedianOfTwoSortedArraysBenchmark)),
// });
//BenchmarkRunner.Run(
// new[]
// {
// BenchmarkConverter.TypeToBenchmarks(typeof(IsNumericBenchmark)),
// });
//BenchmarkRunner.Run(
// new[]
// {
// BenchmarkConverter.TypeToBenchmarks(typeof(EncryptBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(RecyclableEncryptionBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(RecyclableAllocationBenchmark)),
// });
//BenchmarkRunner.Run(
// new[]
// {
// BenchmarkConverter.TypeToBenchmarks(typeof(RecyclableTransformBenchmark)),
// BenchmarkConverter.TypeToBenchmarks(typeof(RecyclableTransformBenchmark2)),
// BenchmarkConverter.TypeToBenchmarks(typeof(RecyclableTransformBenchmark3)),
// BenchmarkConverter.TypeToBenchmarks(typeof(RecyclableTransformBenchmark4)),
// });
//BenchmarkRunner.Run(
// new[]
// {
// BenchmarkConverter.TypeToBenchmarks(typeof(RecyclableTransformBenchmark3)),
// BenchmarkConverter.TypeToBenchmarks(typeof(AllocationEncryptionBenchmark)),
// });
BenchmarkRunner.Run(
new[]
{
BenchmarkConverter.TypeToBenchmarks(typeof(GzipAllocationBenchmark)),
});
}
private static Benchmark CreateBenchmark(IConfig config)
{
return(BenchmarkConverter.TypeToBenchmarks(typeof(MockBenchmarkClass), config).First());
}
private IList <Benchmark> CreateBenchmarks(Type targetBenchmarkType)
{
return(BenchmarkConverter.TypeToBenchmarks(targetBenchmarkType));
}
private static void AssertConsistent <TBenchmark>()
{
var validationErrors = ReturnValueValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(TBenchmark))).ToList();
Assert.Empty(validationErrors);
}
private IEnumerable <Type> EnumerateBenchmarks(IConfig config)
=> from type in Assembly.GetTypes()
where !type.IsGenericType
where BenchmarkConverter.TypeToBenchmarks(type, config).BenchmarksCases.Length != 0
select type;
public void Benchmark_Now_against_DateTimeOffset()
{
BenchmarkRunnerCore.Run(
BenchmarkConverter.TypeToBenchmarks(typeof(PreciseDateTimeBenchmark_Now)),
job => new InProcessToolchain(false));
}
private Benchmark[] CreateBenchmarks(Type targetBenchmarkType)
{
return(BenchmarkConverter.TypeToBenchmarks(targetBenchmarkType).Benchmarks);
}
private static BenchmarkReport[] CreateReports(IConfig config)
{
BenchmarkRunInfo benchmarkRunInfo = BenchmarkConverter.TypeToBenchmarks(typeof(MockBenchmarkClass), config);
return(benchmarkRunInfo.BenchmarksCases.Select(CreateReport).ToArray());
}
private IEnumerable <Type> EnumerateBenchmarks(IConfig config)
=> from type in GetType().GetTypeInfo().Assembly.GetTypes()
where !type.IsGenericType
where type.Namespace?.StartsWith(BenchmarksBaseNamespace) ?? false
where BenchmarkConverter.TypeToBenchmarks(type, config).BenchmarksCases.Length != 0
select type;
private static BenchmarkCase[] CreateBenchmarks <TBenchmarks>(IConfig config) => BenchmarkConverter.TypeToBenchmarks(typeof(TBenchmarks), config).BenchmarksCases;
private static int Run(Options options)
{
var job = new Job();
job.Run.RunStrategy = RunStrategy.Monitoring;
job.Run.LaunchCount = 1;
job.Run.WarmupCount = options.WarmupCount;
job.Run.TargetCount = options.TargetCount;
// Increase timeout from default 5 minutes to 10 minutes. Required for OrchardCore.
job = job.With(new InProcessToolchain(timeout: TimeSpan.FromMinutes(10), codegenMode: BenchmarkActionCodegen.ReflectionEmit, logOutput: true));
var config = (IConfig)ManualConfig.Create(DefaultConfig.Instance);
((ManualConfig)config).Set(new NoOpOrderProvider());
config = config.With(job);
if (options.Debug)
{
((List <IValidator>)config.GetValidators()).Remove(JitOptimizationsValidator.FailOnError);
config = config.With(JitOptimizationsValidator.DontFailOnError);
}
var allBenchmarks = new List <Benchmark>();
foreach (var type in typeof(Program).Assembly.GetTypes().Where(t => !t.IsAbstract).Where(t => t.IsPublic))
{
allBenchmarks.AddRange(BenchmarkConverter.TypeToBenchmarks(type, config).Benchmarks);
}
var selectedBenchmarks = (IEnumerable <Benchmark>)allBenchmarks;
var parameters = ParametersToDictionary(options.Parameters);
// If not specified, default "Restore" to "true" for Core and "false" for Framework,
// to match typical customer usage.
if (!parameters.ContainsKey("Restore"))
{
selectedBenchmarks = selectedBenchmarks.Where(b =>
{
if (b.Target.Type.Name.IndexOf("Core", StringComparison.OrdinalIgnoreCase) >= 0)
{
return((bool)b.Parameters["Restore"]);
}
else if (b.Target.Type.Name.IndexOf("Framework", StringComparison.OrdinalIgnoreCase) >= 0)
{
return(!(bool)b.Parameters["Restore"]);
}
else
{
return(true);
}
});
}
// If not specified, default "Parallel" to "true" to match typical customer usage.
if (!parameters.ContainsKey("Parallel"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => (bool?)b.Parameters["Parallel"] ?? true);
}
// If not specified, default "MSBuildFlavor" to "Core" for Core, to match typical customer usage.
if (!parameters.ContainsKey("MSBuildFlavor"))
{
selectedBenchmarks = selectedBenchmarks.Where(b =>
{
if (b.Target.Type.Name.IndexOf("Core", StringComparison.OrdinalIgnoreCase) >= 0)
{
return(((MSBuildFlavor)b.Parameters["MSBuildFlavor"]) == MSBuildFlavor.Core);
}
else
{
return(true);
}
});
}
// If MSBuildFlavor=Core, MSBuildVersion is irrelevant, so limit it to "NotApplicable"
selectedBenchmarks = selectedBenchmarks.Where(b =>
!(((MSBuildFlavor?)b.Parameters["MSBuildFlavor"]) == MSBuildFlavor.Core &&
!b.Parameters["MSBuildVersion"].ToString().Equals("NotApplicable", StringComparison.OrdinalIgnoreCase)));
// If MSBuildFlavor=Framework or type is Framework, MSBuildVersion is required, so skip "NotApplicable"
selectedBenchmarks = selectedBenchmarks.Where(b =>
!((((MSBuildFlavor?)b.Parameters["MSBuildFlavor"]) == MSBuildFlavor.Framework ||
b.Target.Type.Name.IndexOf("Framework", StringComparison.OrdinalIgnoreCase) >= 0) &&
b.Parameters["MSBuildVersion"].ToString().Equals("NotApplicable", StringComparison.OrdinalIgnoreCase)));
// If not specified, limit "MSBuildVersion" to "15.6" or "NotApplicable"
if (!parameters.ContainsKey("MSBuildVersion"))
{
selectedBenchmarks = selectedBenchmarks.Where(b =>
b.Parameters["MSBuildVersion"] == null ||
((string)b.Parameters["MSBuildVersion"]).StartsWith("15.6", StringComparison.OrdinalIgnoreCase) ||
((string)b.Parameters["MSBuildVersion"]).Equals("NotApplicable", StringComparison.OrdinalIgnoreCase));
}
// If not specified, default "NodeReuse" to "true" to match typical customer usage.
if (!parameters.ContainsKey("NodeReuse"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => (bool?)b.Parameters["NodeReuse"] ?? true);
}
// Large apps and "SourceChanged" methods can choose from SourceChanged.Leaf and SourceChanged.Root
// All other apps and methods must use SourceChanged.NotApplicable
selectedBenchmarks = selectedBenchmarks.Where(b =>
{
var sourceChanged = ((SourceChanged)b.Parameters["SourceChanged"]);
if (b.Target.Type.Name.IndexOf("Large", StringComparison.OrdinalIgnoreCase) >= 0 &&
b.Target.Method.Name.IndexOf("SourceChanged", StringComparison.OrdinalIgnoreCase) >= 0)
{
return((sourceChanged == SourceChanged.Leaf) || (sourceChanged == SourceChanged.Root));
}
else
{
return(sourceChanged == SourceChanged.NotApplicable);
}
});
// If not specified, remove SourceChanged=Root, since SourceChanged=Leaf is tested more often
if (!parameters.ContainsKey("SourceChanged"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => ((SourceChanged)b.Parameters["SourceChanged"]) != SourceChanged.Root);
}
// If not specified, default RazorCompileOnBuild to false
if (!parameters.ContainsKey("RazorCompileOnBuild"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => !(bool?)b.Parameters["RazorCompileOnBuild"] ?? true);
}
// If not specified, default TargetFramework to 2.0
if (!parameters.ContainsKey("TargetFramework"))
{
selectedBenchmarks = selectedBenchmarks.Where(b =>
((string)b.Parameters["TargetFramework"])?.Equals("2.0", StringComparison.OrdinalIgnoreCase) ?? true);
}
selectedBenchmarks = selectedBenchmarks.
Where(b => !options.Types.Any() ||
b.Target.Type.Name.ContainsAny(options.Types, StringComparison.OrdinalIgnoreCase)).
Where(b => !options.Methods.Any() ||
b.Target.Method.Name.ContainsAny(options.Methods, StringComparison.OrdinalIgnoreCase)).
Where(b => b.Parameters.Match(parameters));
BenchmarkRunner.Run(selectedBenchmarks.ToArray(), config);
return(0);
}
private void AssertBenchmarkName <T>(string expectedBenchmarkName)
{
var benchmark = BenchmarkConverter.TypeToBenchmarks(typeof(T)).BenchmarksCases.Single();
Assert.Equal(expectedBenchmarkName, FullNameProvider.GetBenchmarkName(benchmark));
}
public void NullIsSupportedAsElementOfParamsSource()
{
BenchmarkConverter.TypeToBenchmarks(typeof(ParamsSourceWithNull));
}
private static int Run(Options options)
{
var job = new Job();
job.Run.RunStrategy = RunStrategy.Monitoring;
job.Run.LaunchCount = 1;
job.Run.WarmupCount = options.WarmupCount;
job.Run.TargetCount = options.TargetCount;
// Increase timeout from default 5 minutes to 30 minutes. Required for OrchardCore, especially when using multiple iterations
job = job.With(new InProcessToolchain(timeout: TimeSpan.FromMinutes(30), codegenMode: BenchmarkActionCodegen.ReflectionEmit, logOutput: true));
var config = (IConfig)ManualConfig.Create(DefaultConfig.Instance);
// Replace DefaultOrderProvider with NoOpOrderProvider to improve startup time (we don't care about order benchmarks are run)
((ManualConfig)config).Set(new NoOpOrderProvider());
config = config.With(job);
// Allow running debug build when debugger is attached
if (Debugger.IsAttached)
{
((List <IValidator>)config.GetValidators()).Remove(JitOptimizationsValidator.FailOnError);
config = config.With(JitOptimizationsValidator.DontFailOnError);
}
var allBenchmarks = new List <Benchmark>();
foreach (var type in typeof(Program).Assembly.GetTypes().Where(t => !t.IsAbstract).Where(t => t.IsPublic))
{
allBenchmarks.AddRange(BenchmarkConverter.TypeToBenchmarks(type, config).Benchmarks);
}
var selectedBenchmarks = (IEnumerable <Benchmark>)allBenchmarks;
var parameters = ParametersToDictionary(options.Parameters);
// If not specified, default "Restore" to "true" for Core and "false" for Framework,
// to match typical customer usage.
if (!parameters.ContainsKey("Restore"))
{
selectedBenchmarks = selectedBenchmarks.Where(b =>
{
if (typeof(CoreApp).IsAssignableFrom(b.Target.Type))
{
return((bool)b.Parameters["Restore"]);
}
else if (typeof(FrameworkApp).IsAssignableFrom(b.Target.Type))
{
return(!(bool)b.Parameters["Restore"]);
}
else
{
return(true);
}
});
}
// If not specified, default "PackageManagementFormat" to "PackagesConfig" to match typical ASP.NET customer usage.
if (!parameters.ContainsKey("PackageManagementFormat"))
{
selectedBenchmarks = selectedBenchmarks.Where(b =>
(PackageManagementFormat?)b.Parameters["PackageManagementFormat"] == null ||
(PackageManagementFormat?)b.Parameters["PackageManagementFormat"] == PackageManagementFormat.PackagesConfig);
}
// If not specified, default "Parallel" to "true" to match typical customer usage.
if (!parameters.ContainsKey("Parallel"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => (bool?)b.Parameters["Parallel"] ?? true);
}
// If not specified, default "MSBuildFlavor" to "Core" for Core, to match typical customer usage.
if (!parameters.ContainsKey("MSBuildFlavor"))
{
selectedBenchmarks = selectedBenchmarks.Where(b =>
{
if (typeof(CoreApp).IsAssignableFrom(b.Target.Type))
{
return(((MSBuildFlavor)b.Parameters["MSBuildFlavor"]) == MSBuildFlavor.Core);
}
else
{
return(true);
}
});
}
// If MSBuildFlavor=Core, MSBuildVersion is irrelevant, so limit it to "NotApplicable"
selectedBenchmarks = selectedBenchmarks.Where(b =>
!(((MSBuildFlavor?)b.Parameters["MSBuildFlavor"]) == MSBuildFlavor.Core &&
!b.Parameters["MSBuildVersion"].ToString().Equals("NotApplicable", StringComparison.OrdinalIgnoreCase)));
// If MSBuildFlavor=Framework or type is Framework, MSBuildVersion is required, so skip "NotApplicable"
selectedBenchmarks = selectedBenchmarks.Where(b =>
!((((MSBuildFlavor?)b.Parameters["MSBuildFlavor"]) == MSBuildFlavor.Framework ||
typeof(FrameworkApp).IsAssignableFrom(b.Target.Type)) &&
b.Parameters["MSBuildVersion"].ToString().Equals("NotApplicable", StringComparison.OrdinalIgnoreCase)));
// If not specified, limit "MSBuildVersion" to "15.6" or "NotApplicable"
if (!parameters.ContainsKey("MSBuildVersion"))
{
selectedBenchmarks = selectedBenchmarks.Where(b =>
b.Parameters["MSBuildVersion"] == null ||
((string)b.Parameters["MSBuildVersion"]).StartsWith("15.6", StringComparison.OrdinalIgnoreCase) ||
((string)b.Parameters["MSBuildVersion"]).Equals("NotApplicable", StringComparison.OrdinalIgnoreCase));
}
// If not specified, default "NodeReuse" to "true" to match typical customer usage.
if (!parameters.ContainsKey("NodeReuse"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => (bool?)b.Parameters["NodeReuse"] ?? true);
}
// "SourceChanged" methods on types implementing ISourceChanged can choose from SourceChanged=Leaf/Root SourceChangeType=Implementation/Api.
// All other apps and methods must use SourceChanged=NotApplicable and SourceChangeType=NotApplicable.
selectedBenchmarks = selectedBenchmarks.Where(b =>
{
var sourceChanged = ((SourceChanged)b.Parameters["SourceChanged"]);
var sourceChangeType = ((SourceChangeType)b.Parameters["SourceChangeType"]);
if (typeof(ISourceChanged).IsAssignableFrom(b.Target.Type) &&
b.Target.Method.Name.IndexOf("SourceChanged", StringComparison.OrdinalIgnoreCase) >= 0)
{
return(((sourceChanged == SourceChanged.Leaf) || (sourceChanged == SourceChanged.Root)) &&
((sourceChangeType == SourceChangeType.Implementation) || (sourceChangeType == SourceChangeType.Api)));
}
else
{
return(sourceChanged == SourceChanged.NotApplicable && sourceChangeType == SourceChangeType.NotApplicable);
}
});
// If not specified, remove SourceChanged=Root, since SourceChanged=Leaf is tested more often
if (!parameters.ContainsKey("SourceChanged"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => ((SourceChanged)b.Parameters["SourceChanged"]) != SourceChanged.Root);
}
// If not specified, remove SourceChangeType=Api, since SourceChangeType=Implementation is tested more often
if (!parameters.ContainsKey("SourceChangeType"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => ((SourceChangeType)b.Parameters["SourceChangeType"]) != SourceChangeType.Api);
}
// If not specified, default RazorCompileOnBuild to false
if (!parameters.ContainsKey("RazorCompileOnBuild"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => !(bool?)b.Parameters["RazorCompileOnBuild"] ?? true);
}
// If not specified, default TargetFramework to 2.0
if (!parameters.ContainsKey("TargetFramework"))
{
selectedBenchmarks = selectedBenchmarks.Where(b =>
((string)b.Parameters["TargetFramework"])?.Equals("2.0", StringComparison.OrdinalIgnoreCase) ?? true);
}
// NoBuild should be "false" or "true" for "Run" methods, else "null"
selectedBenchmarks = selectedBenchmarks.Where(b =>
{
if (b.Target.Method.Name.StartsWith("run", StringComparison.OrdinalIgnoreCase))
{
return((bool?)b.Parameters["NoBuild"] == false || (bool?)b.Parameters["NoBuild"] == true);
}
else
{
return((bool?)b.Parameters["NoBuild"] == null);
}
});
// If not specified, default NoBuild to null or false
if (!parameters.ContainsKey("NoBuild"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => (bool?)b.Parameters["NoBuild"] == false || (bool?)b.Parameters["NoBuild"] == null);
}
// If not specified, default TieredJit to null or false
if (!parameters.ContainsKey("TieredJit"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => (bool?)b.Parameters["TieredJit"] == false || (bool?)b.Parameters["TieredJit"] == null);
}
// If not specified, default ProduceReferenceAssembly to null or false
if (!parameters.ContainsKey("ProduceReferenceAssembly"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => (bool?)b.Parameters["ProduceReferenceAssembly"] == false || (bool?)b.Parameters["ProduceReferenceAssembly"] == null);
}
// If not specified, default RestoreUseSkipNonexistentTargets to null or true
if (!parameters.ContainsKey("RestoreUseSkipNonexistentTargets"))
{
selectedBenchmarks = selectedBenchmarks.Where(b => (bool?)b.Parameters["RestoreUseSkipNonexistentTargets"] == true || (bool?)b.Parameters["RestoreUseSkipNonexistentTargets"] == null);
}
selectedBenchmarks = selectedBenchmarks.
Where(b => !options.Types.Any() ||
b.Target.Type.Name.ContainsAny(options.Types, StringComparison.OrdinalIgnoreCase)).
Where(b => !options.Methods.Any() ||
b.Target.Method.Name.ContainsAny(options.Methods, StringComparison.OrdinalIgnoreCase)).
Where(b => b.Parameters.Match(parameters));
BenchmarkRunner.Run(selectedBenchmarks.ToArray(), config);
return(0);
}
private static IEnumerable <Benchmark> CreateBenchmarks(IConfig config) => BenchmarkConverter.TypeToBenchmarks(typeof(MockBenchmarkClass), config).Benchmarks;
private static BenchmarkCase[] CreateBenchmarks(IConfig config) => BenchmarkConverter.TypeToBenchmarks(typeof(MockBenchmarkClass), config).BenchmarksCases;
private Benchmark[] CreateBenchmarks(Type[] types)
{
return(types.SelectMany(type => BenchmarkConverter.TypeToBenchmarks(type).Benchmarks).ToArray());
}
public void PropertyWithoutPublicSetterParamsAreDiscovered()
{
Assert.Throws <InvalidOperationException>(() => ExecutionValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(PropertyWithoutPublicSetterParams))));
}
private BenchmarkCase[] CreateBenchmarks(Type[] types, IConfig config)
{
return(types.SelectMany(type => BenchmarkConverter.TypeToBenchmarks(type, config).BenchmarksCases).ToArray());
}
public void NonFailingBenchmarksAreOmmited()
{
var validationErrors = ExecutionValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(NonFailingBenchmark)));
Assert.Empty(validationErrors);
}
static void Main(string[] args)
{
var summary = BenchmarkRunnerCore.Run(BenchmarkConverter.TypeToBenchmarks(typeof(StackTesting)), t => InProcessToolchain.Instance);
Console.Read();
}
public void MultipleParamsDoNotMultiplySetup()
{
var validationErrors = ExecutionValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(MultipleParamsAndSingleSetup)));
Assert.Empty(validationErrors);
}
static void Main(string[] args)
{
// Pick a benchmark.
var availableBenchmarks = Benchmarks.Benchmarks.All;
if (args.Length == 0)
{
Console.WriteLine("Must provide the name of a benchmark class. (e.g. ./Autofac.BenchmarkProfiling.exe ChildScopeResolveBenchmark)");
Console.WriteLine("Possible benchmarks are:");
PrintBenchmarks(availableBenchmarks);
return;
}
var inputType = args[0];
var selectedBenchmark = availableBenchmarks.FirstOrDefault(x => x.Name.Equals(inputType, StringComparison.InvariantCultureIgnoreCase));
if (selectedBenchmark is null)
{
Console.WriteLine("Specified benchmark does not exist.");
PrintBenchmarks(availableBenchmarks);
return;
}
var benchRunInfo = BenchmarkConverter.TypeToBenchmarks(selectedBenchmark);
BenchmarkCase selectedCase = null;
if (benchRunInfo.BenchmarksCases.Length == 0)
{
Console.WriteLine("No benchmark cases in specified benchmark.");
return;
}
else if (benchRunInfo.BenchmarksCases.Length == 1)
{
selectedCase = benchRunInfo.BenchmarksCases[0];
}
else
{
// Multiple benchmark cases. Has one been supplied?
if (args.Length > 1)
{
if (uint.TryParse(args[1], out var selection))
{
if (selection < benchRunInfo.BenchmarksCases.Length)
{
selectedCase = benchRunInfo.BenchmarksCases[selection];
}
else
{
Console.WriteLine("Invalid benchmark case number provided. Possible options are: ");
PrintCases(benchRunInfo);
}
}
else
{
Console.WriteLine("Cannot parse provided benchmark case selection.");
return;
}
}
else
{
Console.WriteLine("Specified benchmark has multiple possible cases; a single case must be specified. Possible options are:");
PrintCases(benchRunInfo);
return;
}
}
var benchInstance = Activator.CreateInstance(selectedCase.Descriptor.Type);
var setupAction = BenchmarkActionFactory.CreateGlobalSetup(selectedCase.Descriptor, benchInstance);
var cleanupAction = BenchmarkActionFactory.CreateGlobalCleanup(selectedCase.Descriptor, benchInstance);
// Workload method is generated differently when BenchmarkDotNet actually runs; we'll need to wrap it in the set of parameters.
// It's way slower than they way they do it, but it should still give us good profiler results.
Action <int> workloadAction = (repeat) =>
{
while (repeat > 0)
{
selectedCase.Descriptor.WorkloadMethod.Invoke(benchInstance, selectedCase.Parameters.Items.Select(x => x.Value).ToArray());
repeat--;
}
};
setupAction.InvokeSingle();
// Warmup.
workloadAction(100);
// Now start a new thread.
var runThread = new Thread(new ThreadStart(() =>
{
// Do a lot.
workloadAction(10000);
}))
{
Name = "Workload Thread"
};
runThread.Start();
runThread.Join();
cleanupAction.InvokeSingle();
}
