private IConfig CreateInProcessConfig(BenchmarkActionCodegen codegenMode, OutputLogger logger = null, IDiagnoser diagnoser = null)
{
return(new ManualConfig()
.AddJob(Job.Dry.WithToolchain(new InProcessToolchain(TimeSpan.Zero, codegenMode, true)).WithInvocationCount(UnrollFactor).WithUnrollFactor(UnrollFactor))
.AddLogger(logger ?? (Output != null ? new OutputLogger(Output) : ConsoleLogger.Default))
.AddColumnProvider(DefaultColumnProviders.Instance));
}
public BenchmarkActionValueTask(object instance, MethodInfo method, BenchmarkActionCodegen codegenMode, int unrollFactor)
{
bool isOverhead = method == null;
if (!isOverhead)
{
startTaskCallback = CreateWorkload <Func <ValueTask <T> > >(instance, method);
callback = ExecuteBlocking;
}
else
{
callback = Overhead;
}
InvokeSingle = InvokeSingleHardcoded;
if (UseFallbackCode(codegenMode, unrollFactor))
{
unrolledCallback = Unroll(callback, unrollFactor);
InvokeMultiple = InvokeMultipleHardcoded;
}
else
{
InvokeMultiple = EmitInvokeMultiple(this, nameof(callback), nameof(result), unrollFactor);
}
}
public BenchmarkActionTask(object instance, MethodInfo method, BenchmarkActionCodegen codegenMode, int unrollFactor)
{
bool isIdle = method == null;
if (!isIdle)
{
startTaskCallback = CreateMain <Func <Task> >(instance, method);
callback = ExecuteBlocking;
}
else
{
callback = Idle;
}
InvokeSingle = callback;
if (UseFallbackCode(codegenMode, unrollFactor))
{
unrolledCallback = Unroll(callback, unrollFactor);
InvokeMultiple = InvokeMultipleHardcoded;
}
else
{
InvokeMultiple = EmitInvokeMultiple(this, nameof(callback), null, unrollFactor);
}
}
public static int Run(IHostApi hostApi, Benchmark benchmark, BenchmarkActionCodegen codegenMode)
{
bool isDiagnoserAttached = hostApi.IsDiagnoserAttached;
// the first thing to do is to let diagnosers hook in before anything happens
// so all jit-related diagnosers can catch first jit compilation!
if (isDiagnoserAttached)
{
hostApi.BeforeAnythingElse();
}
try
{
// we are not using Runnable here in any direct way in order to avoid strong dependency Main<=>Runnable
// which could cause the jitting/assembly loading to happen before we do anything
// we have some jitting diagnosers and we want them to catch all the informations!!
var inProcessRunnableTypeName = $"{typeof(InProcessRunner).FullName}+{nameof(Runnable)}";
var type = typeof(InProcessRunner).GetTypeInfo().Assembly.GetType(inProcessRunnableTypeName);
if (type == null)
{
throw new InvalidOperationException($"Bug: type {inProcessRunnableTypeName} not found.");
}
type.GetMethod(nameof(Runnable.RunCore), BindingFlags.Public | BindingFlags.Static)
.Invoke(null, new object[] { hostApi, benchmark, codegenMode });
return(0);
}
catch (Exception ex)
{
hostApi.WriteLine(ex.ToString());
return(-1);
}
}
/// <summary>
/// Dispatch method that creates <see cref="BenchmarkAction"/> using
/// <paramref name="targetMethod"/> or <paramref name="fallbackIdleSignature"/> to find correct implementation.
/// Either <paramref name="targetMethod"/> or <paramref name="fallbackIdleSignature"/> should be not <c>null</c>.
/// </summary>
private static BenchmarkAction CreateCore(
[NotNull] object instance,
[CanBeNull] MethodInfo targetMethod,
[CanBeNull] MethodInfo fallbackIdleSignature,
BenchmarkActionCodegen codegenMode,
int unrollFactor)
{
Type resultType;
object resultInstance;
PrepareInstanceAndResultType(
instance, targetMethod, fallbackIdleSignature, out resultInstance, out resultType);
if (resultType == typeof(void))
{
return(new BenchmarkActionVoid(resultInstance, targetMethod, codegenMode, unrollFactor));
}
if (resultType == typeof(Task))
{
return(new BenchmarkActionTask(resultInstance, targetMethod, codegenMode, unrollFactor));
}
if (resultType.GetTypeInfo().IsGenericType)
{
var genericType = resultType.GetGenericTypeDefinition();
var argType = resultType.GenericTypeArguments[0];
if (typeof(Task <>) == genericType)
{
return(Create(
typeof(BenchmarkActionTask <>).MakeGenericType(argType),
resultInstance, targetMethod,
codegenMode, unrollFactor));
}
if (typeof(ValueTask <>).IsAssignableFrom(genericType))
{
return(Create(
typeof(BenchmarkActionValueTask <>).MakeGenericType(argType),
resultInstance, targetMethod,
codegenMode, unrollFactor));
}
}
if (targetMethod == null && resultType.GetTypeInfo().IsValueType)
{
// for Idle: we return int because creating bigger ValueType could take longer than benchmarked method itself.
resultType = typeof(int);
}
return(Create(
typeof(BenchmarkAction <>).MakeGenericType(resultType),
resultInstance, targetMethod,
codegenMode, unrollFactor));
}
/// <summary>Initializes a new instance of the <see cref="InProcessExecutor" /> class.</summary>
/// <param name="timeout">Timeout for the run.</param>
/// <param name="codegenMode">Describes how benchmark action code is generated.</param>
/// <param name="logOutput"><c>true</c> if the output should be logged.</param>
public InProcessExecutor(TimeSpan timeout, BenchmarkActionCodegen codegenMode, bool logOutput)
{
if (timeout == TimeSpan.Zero)
{
timeout = DefaultTimeout;
}
ExecutionTimeout = timeout;
CodegenMode = codegenMode;
LogOutput = logOutput;
}
public BenchmarkAction(object instance, MethodInfo method, BenchmarkActionCodegen codegenMode, int unrollFactor)
{
callback = CreateMainOrIdle <Func <T> >(instance, method, IdleStatic, IdleInstance);
InvokeSingle = InvokeSingleHardcoded;
if (UseFallbackCode(codegenMode, unrollFactor))
{
unrolledCallback = Unroll(callback, unrollFactor);
InvokeMultiple = InvokeMultipleHardcoded;
}
else
{
InvokeMultiple = EmitInvokeMultiple(this, nameof(callback), nameof(result), unrollFactor);
}
}
public BenchmarkActionVoid(object instance, MethodInfo method, BenchmarkActionCodegen codegenMode, int unrollFactor)
{
callback = CreateWorkloadOrOverhead <Action>(instance, method, OverheadStatic, OverheadInstance);
InvokeSingle = callback;
if (UseFallbackCode(codegenMode, unrollFactor))
{
unrolledCallback = Unroll(callback, unrollFactor);
InvokeMultiple = InvokeMultipleHardcoded;
}
else
{
InvokeMultiple = EmitInvokeMultiple(this, nameof(callback), null, unrollFactor);
}
}
public BenchmarkActionVoid(
object instance, MethodInfo method, BenchmarkActionCodegen codegenMode, int unrollFactor)
{
_callback = CreateMainOrIdle <Action>(instance, method, IdleStatic, IdleInstance);
InvokeSingle = _callback;
if (UseFallbackCode(codegenMode, unrollFactor))
{
_unrolledCallback = Unroll(_callback, unrollFactor);
InvokeMultiple = InvokeMultipleHardcoded;
}
else
{
InvokeMultiple = EmitInvokeMultiple(this, nameof(_callback), null, unrollFactor);
}
}
public static int Run(IHost host, BenchmarkCase benchmarkCase, BenchmarkActionCodegen codegenMode, IConfig config)
{
// the first thing to do is to let diagnosers hook in before anything happens
// so all jit-related diagnosers can catch first jit compilation!
host.BeforeAnythingElse();
try
{
// we are not using Runnable here in any direct way in order to avoid strong dependency Main<=>Runnable
// which could cause the jitting/assembly loading to happen before we do anything
// we have some jitting diagnosers and we want them to catch all the informations!!
var inProcessRunnableTypeName = $"{typeof(InProcessRunner).FullName}+{nameof(Runnable)}";
var type = typeof(InProcessRunner).GetTypeInfo().Assembly.GetType(inProcessRunnableTypeName);
if (type == null)
{
throw new InvalidOperationException($"Bug: type {inProcessRunnableTypeName} not found.");
}
type.GetMethod(nameof(Runnable.RunCore), BindingFlags.Public | BindingFlags.Static)
.Invoke(null, new object[] { host, benchmarkCase, codegenMode, config });
return(0);
}
catch (Exception oom) when(oom is OutOfMemoryException || oom is TargetInvocationException reflection && reflection.InnerException is OutOfMemoryException)
{
host.WriteLine();
host.WriteLine("OutOfMemoryException!");
host.WriteLine("BenchmarkDotNet continues to run additional iterations until desired accuracy level is achieved. It's possible only if the benchmark method doesn't have any side-effects.");
host.WriteLine("If your benchmark allocates memory and keeps it alive, you are creating a memory leak.");
host.WriteLine("You should redesign your benchmark and remove the side-effects. You can use `OperationsPerInvoke`, `IterationSetup` and `IterationCleanup` to do that.");
host.WriteLine();
host.WriteLine(oom.ToString());
return(-1);
}
catch (Exception ex)
{
host.WriteLine();
host.WriteLine(ex.ToString());
return(-1);
}
finally
{
host.AfterAll();
}
}
public BenchmarkActionValueTask(
object instance, MethodInfo method, BenchmarkActionCodegen codegenMode, int unrollFactor)
{
_startTaskCallback = CreateMainOrIdle <Func <ValueTask <T> > >(instance, method, IdleStatic, IdleInstance);
_callback = ExecuteBlocking;
InvokeSingle = InvokeSingleHardcoded;
if (UseFallbackCode(codegenMode, unrollFactor))
{
_unrolledCallback = Unroll(_callback, unrollFactor);
InvokeMultiple = InvokeMultipleHardcoded;
}
else
{
InvokeMultiple = EmitInvokeMultiple(this, nameof(_callback), nameof(_result), unrollFactor);
}
}
public static void RunCore(IHost host, BenchmarkCase benchmarkCase, BenchmarkActionCodegen codegenMode, IConfig config)
{
var target = benchmarkCase.Descriptor;
var job = benchmarkCase.Job; // TODO: filter job (same as SourceCodePresenter does)?
int unrollFactor = benchmarkCase.Job.ResolveValue(RunMode.UnrollFactorCharacteristic, EnvironmentResolver.Instance);
// DONTTOUCH: these should be allocated together
var instance = Activator.CreateInstance(benchmarkCase.Descriptor.Type);
var workloadAction = BenchmarkActionFactory.CreateWorkload(target, instance, codegenMode, unrollFactor);
var overheadAction = BenchmarkActionFactory.CreateOverhead(target, instance, codegenMode, unrollFactor);
var globalSetupAction = BenchmarkActionFactory.CreateGlobalSetup(target, instance);
var globalCleanupAction = BenchmarkActionFactory.CreateGlobalCleanup(target, instance);
var iterationSetupAction = BenchmarkActionFactory.CreateIterationSetup(target, instance);
var iterationCleanupAction = BenchmarkActionFactory.CreateIterationCleanup(target, instance);
var dummy1 = BenchmarkActionFactory.CreateDummy();
var dummy2 = BenchmarkActionFactory.CreateDummy();
var dummy3 = BenchmarkActionFactory.CreateDummy();
FillMembers(instance, benchmarkCase);
host.WriteLine();
foreach (string infoLine in BenchmarkEnvironmentInfo.GetCurrent().ToFormattedString())
{
host.WriteLine("// {0}", infoLine);
}
host.WriteLine("// Job: {0}", job.DisplayInfo);
host.WriteLine();
var engineParameters = new EngineParameters
{
Host = host,
WorkloadActionNoUnroll = invocationCount =>
{
for (int i = 0; i < invocationCount; i++)
{
workloadAction.InvokeSingle();
}
},
WorkloadActionUnroll = workloadAction.InvokeMultiple,
Dummy1Action = dummy1.InvokeSingle,
Dummy2Action = dummy2.InvokeSingle,
Dummy3Action = dummy3.InvokeSingle,
OverheadActionNoUnroll = invocationCount =>
{
for (int i = 0; i < invocationCount; i++)
{
overheadAction.InvokeSingle();
}
},
OverheadActionUnroll = overheadAction.InvokeMultiple,
GlobalSetupAction = globalSetupAction.InvokeSingle,
GlobalCleanupAction = globalCleanupAction.InvokeSingle,
IterationSetupAction = iterationSetupAction.InvokeSingle,
IterationCleanupAction = iterationCleanupAction.InvokeSingle,
TargetJob = job,
OperationsPerInvoke = target.OperationsPerInvoke,
MeasureGcStats = config.HasMemoryDiagnoser(),
BenchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase)
};
using (var engine = job
.ResolveValue(InfrastructureMode.EngineFactoryCharacteristic, InfrastructureResolver.Instance)
.CreateReadyToRun(engineParameters))
{
var results = engine.Run();
host.ReportResults(results); // printing costs memory, do this after runs
}
}
protected static bool UseFallbackCode(BenchmarkActionCodegen codegenMode, int unrollFactor) => unrollFactor <= 1 || codegenMode == BenchmarkActionCodegen.DelegateCombine;
/// <summary>Initializes a new instance of the <see cref="InProcessToolchain" /> class.</summary>
/// <param name="timeout">Timeout for the run.</param>
/// <param name="codegenMode">Describes how benchmark action code is generated.</param>
/// <param name="logOutput"><c>true</c> if the output should be logged.</param>
public InProcessToolchain(TimeSpan timeout, BenchmarkActionCodegen codegenMode, bool logOutput)
{
Generator = new InProcessGenerator();
Builder = new InProcessBuilder();
Executor = new InProcessExecutor(timeout, codegenMode, logOutput);
}
/// <summary>Creates idle benchmark action.</summary> /// <param name="target">Target info.</param> /// <param name="instance">Instance of target.</param> /// <param name="codegenMode">Describes how benchmark action code is generated.</param> /// <param name="unrollFactor">Unroll factor.</param> /// <returns>Idle benchmark action.</returns> public static BenchmarkAction CreateIdle(Target target, object instance, BenchmarkActionCodegen codegenMode, int unrollFactor) => CreateCore(instance, null, target.Method, codegenMode, unrollFactor);
public static void RunCore(IHostApi hostApi, Benchmark benchmark, BenchmarkActionCodegen codegenMode)
{
var target = benchmark.Target;
var job = benchmark.Job; // TODO: filter job (same as SourceCodePresenter does)?
var unrollFactor = benchmark.Job.ResolveValue(RunMode.UnrollFactorCharacteristic, EnvResolver.Instance);
// var dummyUnrollFactor = 1 << 6; // TODO: as arg to CreateDummy()?
// DONTTOUCH: these should be allocated together
var instance = Activator.CreateInstance(benchmark.Target.Type);
var mainAction = BenchmarkActionFactory.CreateRun(target, instance, codegenMode, unrollFactor);
var idleAction = BenchmarkActionFactory.CreateIdle(target, instance, codegenMode, unrollFactor);
var setupAction = BenchmarkActionFactory.CreateSetup(target, instance);
var cleanupAction = BenchmarkActionFactory.CreateCleanup(target, instance);
var dummy1 = BenchmarkActionFactory.CreateDummy();
var dummy2 = BenchmarkActionFactory.CreateDummy();
var dummy3 = BenchmarkActionFactory.CreateDummy();
FillProperties(instance, benchmark);
hostApi.WriteLine();
foreach (var infoLine in BenchmarkEnvironmentInfo.GetCurrent().ToFormattedString())
{
hostApi.WriteLine("// {0}", infoLine);
}
hostApi.WriteLine("// Job: {0}", job.DisplayInfo);
hostApi.WriteLine();
var engineParameters = new EngineParameters
{
//HostApi = hostApi,
IsDiagnoserAttached = hostApi.IsDiagnoserAttached,
MainAction = mainAction.InvokeMultiple,
Dummy1Action = dummy1.InvokeSingle,
Dummy2Action = dummy2.InvokeSingle,
Dummy3Action = dummy3.InvokeSingle,
IdleAction = idleAction.InvokeMultiple,
SetupAction = setupAction.InvokeSingle,
CleanupAction = cleanupAction.InvokeSingle,
TargetJob = job,
OperationsPerInvoke = target.OperationsPerInvoke
};
var engine = job
.ResolveValue(InfrastructureMode.EngineFactoryCharacteristic, InfrastructureResolver.Instance)
.Create(engineParameters);
engine.PreAllocate();
setupAction.InvokeSingle();
if (job.ResolveValue(RunMode.RunStrategyCharacteristic, EngineResolver.Instance)
!= RunStrategy.ColdStart)
{
engine.Jitting(); // does first call to main action, must be executed after setup()!
}
if (hostApi.IsDiagnoserAttached)
{
hostApi.AfterSetup();
}
var results = engine.Run();
if (hostApi.IsDiagnoserAttached)
{
hostApi.BeforeCleanup();
}
cleanupAction.InvokeSingle();
hostApi.Print(results); // printing costs memory, do this after runs
}
/// <summary>Creates idle benchmark action.</summary> /// <param name="descriptor">Descriptor info.</param> /// <param name="instance">Instance of target.</param> /// <param name="codegenMode">Describes how benchmark action code is generated.</param> /// <param name="unrollFactor">Unroll factor.</param> /// <returns>Idle benchmark action.</returns> public static BenchmarkAction CreateOverhead(Descriptor descriptor, object instance, BenchmarkActionCodegen codegenMode, int unrollFactor) => CreateCore(instance, null, descriptor.WorkloadMethod, codegenMode, unrollFactor);
/// <summary>Helper to enforce .ctor signature.</summary> private static BenchmarkActionBase Create(Type actionType, object instance, MethodInfo method, BenchmarkActionCodegen codegenMode, int unrollFactor) => (BenchmarkActionBase)Activator.CreateInstance(actionType, instance, method, codegenMode, unrollFactor);
public static void RunCore(IHost host, Benchmark benchmark, BenchmarkActionCodegen codegenMode, IConfig config)
{
var target = benchmark.Target;
var job = benchmark.Job; // TODO: filter job (same as SourceCodePresenter does)?
var unrollFactor = benchmark.Job.ResolveValue(RunMode.UnrollFactorCharacteristic, EnvResolver.Instance);
// DONTTOUCH: these should be allocated together
var instance = Activator.CreateInstance(benchmark.Target.Type);
var mainAction = BenchmarkActionFactory.CreateRun(target, instance, codegenMode, unrollFactor);
var idleAction = BenchmarkActionFactory.CreateIdle(target, instance, codegenMode, unrollFactor);
var globalSetupAction = BenchmarkActionFactory.CreateGlobalSetup(target, instance);
var globalCleanupAction = BenchmarkActionFactory.CreateGlobalCleanup(target, instance);
var iterationSetupAction = BenchmarkActionFactory.CreateIterationSetup(target, instance);
var iterationCleanupAction = BenchmarkActionFactory.CreateIterationCleanup(target, instance);
var dummy1 = BenchmarkActionFactory.CreateDummy();
var dummy2 = BenchmarkActionFactory.CreateDummy();
var dummy3 = BenchmarkActionFactory.CreateDummy();
FillMembers(instance, benchmark);
host.WriteLine();
foreach (var infoLine in BenchmarkEnvironmentInfo.GetCurrent().ToFormattedString())
{
host.WriteLine("// {0}", infoLine);
}
host.WriteLine("// Job: {0}", job.DisplayInfo);
host.WriteLine();
var engineParameters = new EngineParameters
{
Host = host,
MainAction = mainAction.InvokeMultiple,
Dummy1Action = dummy1.InvokeSingle,
Dummy2Action = dummy2.InvokeSingle,
Dummy3Action = dummy3.InvokeSingle,
IdleAction = idleAction.InvokeMultiple,
GlobalSetupAction = globalSetupAction.InvokeSingle,
GlobalCleanupAction = globalCleanupAction.InvokeSingle,
IterationSetupAction = iterationSetupAction.InvokeSingle,
IterationCleanupAction = iterationCleanupAction.InvokeSingle,
TargetJob = job,
OperationsPerInvoke = target.OperationsPerInvoke,
MeasureGcStats = config.HasMemoryDiagnoser()
};
var engine = job
.ResolveValue(InfrastructureMode.EngineFactoryCharacteristic, InfrastructureResolver.Instance)
.Create(engineParameters);
globalSetupAction.InvokeSingle();
iterationSetupAction.InvokeSingle();
if (job.ResolveValue(RunMode.RunStrategyCharacteristic, EngineResolver.Instance).NeedsJitting())
{
engine.Jitting(); // does first call to main action, must be executed after setup()!
}
iterationCleanupAction.InvokeSingle();
var results = engine.Run();
globalCleanupAction.InvokeSingle();
host.ReportResults(results); // printing costs memory, do this after runs
}
