/// <summary>
/// Executes the tests
/// </summary>
/// <returns>True if all tests passed.</returns>
public TestRunnerResult Execute()
{
// Perform core / thread optimizations if we're running in single-threaded mode
// But not if the user has specified that they're going to be running multi-threaded benchmarks
SetProcessPriority(_package.Concurrent);
IBenchmarkOutput output = CreateOutput();
var discovery = new ReflectionDiscovery(output);
var result = new TestRunnerResult()
{
AllTestsPassed = true
};
try
{
foreach (var testFile in _package.Files)
{
var assembly = AssemblyRuntimeLoader.LoadAssembly(testFile);
var benchmarks = discovery.FindBenchmarks(assembly);
foreach (var benchmark in benchmarks)
{
// verify if the benchmark should be included/excluded from the list of benchmarks to be run
if (_package.ShouldRunBenchmark(benchmark.BenchmarkName))
{
output.WriteLine($"------------ STARTING {benchmark.BenchmarkName} ---------- ");
benchmark.Run();
benchmark.Finish();
// if one assert fails, all fail
result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed;
output.WriteLine($"------------ FINISHED {benchmark.BenchmarkName} ---------- ");
result.ExecutedTestsCount = result.ExecutedTestsCount + 1;
}
else
{
output.WriteLine($"------------ NOTRUN {benchmark.BenchmarkName} ---------- ");
result.IgnoredTestsCount = result.IgnoredTestsCount + 1;
}
}
}
}
catch (Exception ex)
{
output.Error(ex, "Error while executing the tests.");
result.AllTestsPassed = false;
}
return(result);
}
/// <summary>
/// NBench Runner takes the following <see cref="args"/>
///
/// C:\> NBench.Runner.exe [assembly name] [output-directory={dir-path}]
///
/// </summary>
/// <param name="args">The commandline arguments</param>
static int Main(string[] args)
{
Output = new CompositeBenchmarkOutput(new ConsoleBenchmarkOutput(), new MarkdownBenchmarkOutput(CommandLine.GetProperty("output-directory")));
Discovery = new ReflectionDiscovery(Output);
string assemblyPath = Path.GetFullPath(args[0]);
// TODO: See issue https://github.com/petabridge/NBench/issues/3
var assembly = AssemblyRuntimeLoader.LoadAssembly(assemblyPath);
/*
* Set processor affinity
*/
Process Proc = Process.GetCurrentProcess();
Proc.ProcessorAffinity = new IntPtr(2); // either of the first two processors
/*
* Set priority
*/
Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.High;
Thread.CurrentThread.Priority = ThreadPriority.Highest;
var benchmarks = Discovery.FindBenchmarks(assembly);
bool anyAssertFailures = false;
foreach (var benchmark in benchmarks)
{
Output.WriteLine($"------------ STARTING {benchmark.BenchmarkName} ---------- ");
benchmark.Run();
benchmark.Finish();
// if one assert fails, all fail
anyAssertFailures = anyAssertFailures || !benchmark.AllAssertsPassed;
Output.WriteLine($"------------ FINISHED {benchmark.BenchmarkName} ---------- ");
}
return(anyAssertFailures ? -1 : 0);
}
/// <summary>
/// Executes the tests
/// </summary>
/// <returns>True if all tests passed.</returns>
public TestRunnerResult Execute()
{
// Perform core / thread optimizations if we're running in single-threaded mode
// But not if the user has specified that they're going to be running multi-threaded benchmarks
SetProcessPriority(_package.Concurrent);
// pass in the runner settings so we can include them in benchmark reports
// also, toggles tracing on or off
var runnerSettings = new RunnerSettings()
{
ConcurrentModeEnabled = _package.Concurrent,
TracingEnabled = _package.Tracing
};
IBenchmarkOutput output = CreateOutput();
var discovery = new ReflectionDiscovery(output,
DefaultBenchmarkAssertionRunner.Instance, // one day we might be able to pass in custom assertion runners, hence why this is here
runnerSettings);
var result = new TestRunnerResult()
{
AllTestsPassed = true
};
try
{
foreach (var testFile in _package.Files)
{
var assembly = AssemblyRuntimeLoader.LoadAssembly(testFile);
var benchmarks = discovery.FindBenchmarks(assembly);
foreach (var benchmark in benchmarks)
{
// verify if the benchmark should be included/excluded from the list of benchmarks to be run
if (_package.ShouldRunBenchmark(benchmark.BenchmarkName))
{
output.StartBenchmark(benchmark.BenchmarkName);
benchmark.Run();
benchmark.Finish();
// if one assert fails, all fail
result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed;
output.FinishBenchmark(benchmark.BenchmarkName);
result.ExecutedTestsCount = result.ExecutedTestsCount + 1;
}
else
{
output.SkipBenchmark(benchmark.BenchmarkName);
result.IgnoredTestsCount = result.IgnoredTestsCount + 1;
}
}
}
}
catch (Exception ex)
{
output.Error(ex, "Error while executing the tests.");
result.AllTestsPassed = false;
}
return(result);
}
