public NetCoreAssemblyRuntimeLoader(string path, IBenchmarkOutput trace)
{
_trace = trace;
if (!File.Exists(path))
{
trace.Error($"[NetCoreAssemblyRuntimeLoader] Unable to find requested assembly [{path}]");
return;
}
Assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(path);
if (Assembly == null)
{
trace.Error($"[NetCoreAssemblyRuntimeLoader] Found assembly [{path}], but was unable to load it.");
return;
}
_dependencyContext = DependencyContext.Load(Assembly);
_loadContext = AssemblyLoadContext.GetLoadContext(Assembly);
_resolver = new CompositeCompilationAssemblyResolver(new ICompilationAssemblyResolver[] {
new AppBaseCompilationAssemblyResolver(Path.GetDirectoryName(path)),
new ReferenceAssemblyPathResolver(),
new PackageCompilationAssemblyResolver()
});
_loadContext.Resolving += LoadContextOnResolving;
_referencedAssemblies = new Lazy <Assembly[]>(LoadReferencedAssemblies);
}
public static IEnumerable <Type> LoadAllTypeConfigurators(Assembly assembly, IBenchmarkOutput output = null)
{
using (var loader = AssemblyRuntimeLoader.WrapAssembly(assembly, output))
{
return(LoadAllTypeConfigurators(loader));
}
}
public AssemblyRuntimeLoader(Assembly assembly, IBenchmarkOutput trace)
{
_trace = trace;
Assembly = assembly;
_binaryDirectory = Path.GetDirectoryName(Assembly.CodeBase);
AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve;
_referencedAssemblies = new Lazy <Assembly[]>(LoadReferencedAssemblies);
}
public Benchmark(BenchmarkSettings settings, IBenchmarkInvoker invoker, IBenchmarkOutput writer)
{
Settings = settings;
_pendingIterations = Settings.NumberOfIterations;
Invoker = invoker;
Output = writer;
CompletedRuns = new Queue <BenchmarkRunReport>(Settings.NumberOfIterations);
Builder = new BenchmarkBuilder(Settings);
}
/// <summary>
/// Loads an assembly into the current AppDomain
/// </summary>
/// <param name="assembly">An already-loaded assembly.</param>
/// <param name="trace">Optional. Benchmark tracing system.</param>
/// <returns>An <see cref="IAssemblyLoader"/> with a reference to the <see cref="Assembly"/> at the specified location.</returns>
public static IAssemblyLoader WrapAssembly(Assembly assembly, IBenchmarkOutput trace = null)
{
trace = trace ?? NoOpBenchmarkOutput.Instance;
#if CORECLR
return(new NetCoreAssemblyRuntimeLoader(assembly, trace));
#else
return(new NetFrameworkAssemblyRuntimeLoader(assembly, trace));
#endif
}
public Benchmark(BenchmarkSettings settings, IBenchmarkInvoker invoker, IBenchmarkOutput writer)
{
Settings = settings;
_pendingIterations = Settings.NumberOfIterations;
Invoker = invoker;
Output = writer;
CompletedRuns = new Queue<BenchmarkRunReport>(Settings.NumberOfIterations);
Builder = new BenchmarkBuilder(Settings);
}
/// <summary>
/// Executes the tests
/// </summary>
/// <returns>True if all tests passed.</returns>
public TestRunnerResult Execute()
{
// Perform core / thread optimizations if we're running in single-threaded mode
// But not if the user has specified that they're going to be running multi-threaded benchmarks
SetProcessPriority(_package.Concurrent);
IBenchmarkOutput output = CreateOutput();
var discovery = new ReflectionDiscovery(output);
var result = new TestRunnerResult()
{
AllTestsPassed = true
};
try
{
foreach (var testFile in _package.Files)
{
var assembly = AssemblyRuntimeLoader.LoadAssembly(testFile);
var benchmarks = discovery.FindBenchmarks(assembly);
foreach (var benchmark in benchmarks)
{
// verify if the benchmark should be included/excluded from the list of benchmarks to be run
if (_package.ShouldRunBenchmark(benchmark.BenchmarkName))
{
output.WriteLine($"------------ STARTING {benchmark.BenchmarkName} ---------- ");
benchmark.Run();
benchmark.Finish();
// if one assert fails, all fail
result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed;
output.WriteLine($"------------ FINISHED {benchmark.BenchmarkName} ---------- ");
result.ExecutedTestsCount = result.ExecutedTestsCount + 1;
}
else
{
output.WriteLine($"------------ NOTRUN {benchmark.BenchmarkName} ---------- ");
result.IgnoredTestsCount = result.IgnoredTestsCount + 1;
}
}
}
}
catch (Exception ex)
{
output.Error(ex, "Error while executing the tests.");
result.AllTestsPassed = false;
}
return(result);
}
public ReflectionDiscovery(IBenchmarkOutput output, IBenchmarkAssertionRunner benchmarkAssertions, RunnerSettings settings)
{
Output = output;
BenchmarkAssertions = benchmarkAssertions;
RunnerSettings = settings;
if (RunnerSettings.TracingEnabled)
{
Trace = new BenchmarkOutputTrace(Output);
}
else
{
Trace = NoOpBenchmarkTrace.Instance;
}
}
public NetCoreAssemblyRuntimeLoader(Assembly assembly, IBenchmarkOutput trace)
{
_trace = trace;
Assembly = assembly;
_dependencyContext = DependencyContext.Load(Assembly);
_loadContext = AssemblyLoadContext.GetLoadContext(Assembly);
_resolver = new CompositeCompilationAssemblyResolver(new ICompilationAssemblyResolver[] {
new AppBaseCompilationAssemblyResolver(Path.GetDirectoryName(Assembly.CodeBase)),
new ReferenceAssemblyPathResolver(),
new PackageCompilationAssemblyResolver()
});
_loadContext.Resolving += LoadContextOnResolving;
_referencedAssemblies = new Lazy <Assembly[]>(LoadReferencedAssemblies);
}
public AssemblyRuntimeLoader(string path, IBenchmarkOutput trace)
{
_trace = trace;
if (!File.Exists(path))
{
trace.Error($"[NetFrameworkAssemblyRuntimeLoader] Unable to find requested assembly [{path}]");
return;
}
Assembly = System.Reflection.Assembly.LoadFile(path);
if (Assembly == null)
{
trace.Error($"[NetFrameworkAssemblyRuntimeLoader] Found assembly [{path}], but was unable to load it.");
return;
}
_binaryDirectory = Path.GetDirectoryName(path);
AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve;
_referencedAssemblies = new Lazy <Assembly[]>(LoadReferencedAssemblies);
}
/// <summary>
/// NBench Runner takes the following <see cref="args"/>
///
/// C:\> NBench.Runner.exe [assembly name] [output-directory={dir-path}]
///
/// </summary>
/// <param name="args">The commandline arguments</param>
static int Main(string[] args)
{
Output = new CompositeBenchmarkOutput(new ConsoleBenchmarkOutput(), new MarkdownBenchmarkOutput(CommandLine.GetProperty("output-directory")));
Discovery = new ReflectionDiscovery(Output);
string assemblyPath = Path.GetFullPath(args[0]);
// TODO: See issue https://github.com/petabridge/NBench/issues/3
var assembly = AssemblyRuntimeLoader.LoadAssembly(assemblyPath);
/*
* Set processor affinity
*/
Process Proc = Process.GetCurrentProcess();
Proc.ProcessorAffinity = new IntPtr(2); // either of the first two processors
/*
* Set priority
*/
Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.High;
Thread.CurrentThread.Priority = ThreadPriority.Highest;
var benchmarks = Discovery.FindBenchmarks(assembly);
bool anyAssertFailures = false;
foreach (var benchmark in benchmarks)
{
Output.WriteLine($"------------ STARTING {benchmark.BenchmarkName} ---------- ");
benchmark.Run();
benchmark.Finish();
// if one assert fails, all fail
anyAssertFailures = anyAssertFailures || !benchmark.AllAssertsPassed;
Output.WriteLine($"------------ FINISHED {benchmark.BenchmarkName} ---------- ");
}
return(anyAssertFailures ? -1 : 0);
}
/// <summary>
/// NBench Runner takes the following <see cref="args"/>
///
/// C:\> NBench.Runner.exe [assembly name] [output-directory={dir-path}]
///
/// </summary>
/// <param name="args">The commandline arguments</param>
static int Main(string[] args)
{
Output = new CompositeBenchmarkOutput(new ConsoleBenchmarkOutput(), new MarkdownBenchmarkOutput(CommandLine.GetProperty("output-directory")));
Discovery = new ReflectionDiscovery(Output);
string assemblyPath = Path.GetFullPath(args[0]);
// TODO: See issue https://github.com/petabridge/NBench/issues/3
var assembly = AssemblyRuntimeLoader.LoadAssembly(assemblyPath);
/*
* Set processor affinity
*/
Process Proc = Process.GetCurrentProcess();
Proc.ProcessorAffinity = new IntPtr(2); // either of the first two processors
/*
* Set priority
*/
Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.High;
Thread.CurrentThread.Priority = ThreadPriority.Highest;
var benchmarks = Discovery.FindBenchmarks(assembly);
bool anyAssertFailures = false;
foreach (var benchmark in benchmarks)
{
Output.WriteLine($"------------ STARTING {benchmark.BenchmarkName} ---------- ");
benchmark.Run();
benchmark.Finish();
// if one assert fails, all fail
anyAssertFailures = anyAssertFailures || !benchmark.AllAssertsPassed;
Output.WriteLine($"------------ FINISHED {benchmark.BenchmarkName} ---------- ");
}
return anyAssertFailures ? -1 : 0;
}
public TestPackage AddOutput(IBenchmarkOutput output)
{
_outputs.Add(output);
return(this);
}
public ReflectionDiscovery(IBenchmarkOutput output, IBenchmarkAssertionRunner benchmarkAssertions)
{
Output = _reflectionOutput = output;
BenchmarkAssertions = benchmarkAssertions;
}
private BenchmarkRunCache(IBenchmarkOutput output)
{
discovery = new ReflectionDiscovery(output);
}
/// <summary>
/// Backwards-compatible constructor for NBench 0.1.6 and earlier.
/// </summary>
/// <param name="settings">The settings for this benchmark.</param>
/// <param name="invoker">The invoker used to execute benchmark and setup / cleanup methods.</param>
/// <param name="writer">The output target this benchmark will write to.</param>
/// <remarks>Uses the <see cref="DefaultBenchmarkAssertionRunner"/> to assert benchmark data.</remarks>
public Benchmark(BenchmarkSettings settings, IBenchmarkInvoker invoker, IBenchmarkOutput writer)
: this(settings, invoker, writer, DefaultBenchmarkAssertionRunner.Instance)
{
}
public ReflectionDiscovery(IBenchmarkOutput output)
{
Output = _reflectionOutput = output;
}
public ReflectionDiscovery(IBenchmarkOutput output) : this(output, DefaultBenchmarkAssertionRunner.Instance, new RunnerSettings() { TracingEnabled = false })
{
}
public BenchmarkOutputTrace(IBenchmarkOutput benchmarkOutput)
{
_benchmarkOutput = benchmarkOutput;
}
public NBenchIntregrationTestWithDependenciesLoadAssembly(ITestOutputHelper output)
{
_output = output;
_benchmarkOutput = new XunitBenchmarkOutputHelper(output);
}
public ReflectionDiscovery(IBenchmarkOutput output)
: this(output, DefaultBenchmarkAssertionRunner.Instance, new RunnerSettings())
{
}
public ReflectionDiscovery(IBenchmarkOutput output, IBenchmarkAssertionRunner benchmarkAssertions, RunnerSettings settings)
{
Output = _reflectionOutput = output;
BenchmarkAssertions = benchmarkAssertions;
RunnerSettings = settings;
if(RunnerSettings.TracingEnabled)
Trace = new BenchmarkOutputTrace(Output);
else
Trace = NoOpBenchmarkTrace.Instance;
}
/// <summary>
/// Backwards-compatible constructor for NBench 0.1.6 and earlier.
/// </summary>
/// <param name="settings">The settings for this benchmark.</param>
/// <param name="invoker">The invoker used to execute benchmark and setup / cleanup methods.</param>
/// <param name="writer">The output target this benchmark will write to.</param>
/// <remarks>Uses the <see cref="DefaultBenchmarkAssertionRunner"/> to assert benchmark data.</remarks>
public Benchmark(BenchmarkSettings settings, IBenchmarkInvoker invoker, IBenchmarkOutput writer)
: this(settings, invoker, writer, DefaultBenchmarkAssertionRunner.Instance)
{
}
/// <summary>
/// Loads an assembly into the current AppDomain
/// </summary>
/// <param name="assemblyPath">The path to an assembly</param>
/// <param name="trace">Optional. Benchmark tracing system.</param>
/// <returns>An <see cref="IAssemblyLoader"/> with a reference to the <see cref="Assembly"/> at the specified location.</returns>
public static IAssemblyLoader LoadAssembly(string assemblyPath, IBenchmarkOutput trace = null)
{
trace = trace ?? NoOpBenchmarkOutput.Instance;
return(new Assemblies.AssemblyRuntimeLoader(assemblyPath, trace));
}
public ReflectionDiscovery(IBenchmarkOutput output)
{
Output = _reflectionOutput = output;
}
public ReflectionDiscovery(IBenchmarkOutput output) : this(output, DefaultBenchmarkAssertionRunner.Instance)
{
}
public ReflectionDiscoveryConfiguratorSpecs(ITestOutputHelper helper)
{
_output = new XunitBenchmarkOutputHelper(helper);
}
/// <summary>
/// Executes the tests
/// </summary>
/// <returns>True if all tests passed.</returns>
public TestRunnerResult Execute()
{
// Perform core / thread optimizations if we're running in single-threaded mode
// But not if the user has specified that they're going to be running multi-threaded benchmarks
SetProcessPriority(_package.Concurrent);
// pass in the runner settings so we can include them in benchmark reports
// also, toggles tracing on or off
var runnerSettings = new RunnerSettings()
{
ConcurrentModeEnabled = _package.Concurrent,
TracingEnabled = _package.Tracing
};
IBenchmarkOutput output = CreateOutput();
var discovery = new ReflectionDiscovery(output,
DefaultBenchmarkAssertionRunner.Instance, // one day we might be able to pass in custom assertion runners, hence why this is here
runnerSettings);
var result = new TestRunnerResult()
{
AllTestsPassed = true
};
try
{
foreach (var testFile in _package.Files)
{
var assembly = AssemblyRuntimeLoader.LoadAssembly(testFile);
var benchmarks = discovery.FindBenchmarks(assembly);
foreach (var benchmark in benchmarks)
{
// verify if the benchmark should be included/excluded from the list of benchmarks to be run
if (_package.ShouldRunBenchmark(benchmark.BenchmarkName))
{
output.StartBenchmark(benchmark.BenchmarkName);
benchmark.Run();
benchmark.Finish();
// if one assert fails, all fail
result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed;
output.FinishBenchmark(benchmark.BenchmarkName);
result.ExecutedTestsCount = result.ExecutedTestsCount + 1;
}
else
{
output.SkipBenchmark(benchmark.BenchmarkName);
result.IgnoredTestsCount = result.IgnoredTestsCount + 1;
}
}
}
}
catch (Exception ex)
{
output.Error(ex, "Error while executing the tests.");
result.AllTestsPassed = false;
}
return(result);
}
/// <summary>
/// Loads an assembly into the current AppDomain
/// </summary>
/// <param name="assembly">An already-loaded assembly.</param>
/// <param name="trace">Optional. Benchmark tracing system.</param>
/// <returns>An <see cref="IAssemblyLoader"/> with a reference to the <see cref="Assembly"/> at the specified location.</returns>
public static IAssemblyLoader WrapAssembly(Assembly assembly, IBenchmarkOutput trace = null)
{
trace = trace ?? NoOpBenchmarkOutput.Instance;
return(new Assemblies.AssemblyRuntimeLoader(assembly, trace));
}
