/// <summary>
/// Initializes a new instance of the test runner.
/// </summary>
/// <param name="package">The test package to be executed</param>
public TestRunner(TestPackage package)
{
_package = package;
_resultsCollector = new ActionBenchmarkOutput(benchmarkAction: f => { _results.Add(f); });
}
/// <summary>
/// Executes the tests
/// </summary>
/// <returns>True if all tests passed.</returns>
public TestRunnerResult Execute()
{
// Perform core / thread optimizations if we're running in single-threaded mode
// But not if the user has specified that they're going to be running multi-threaded benchmarks
SetProcessPriority(_package.Concurrent);
// pass in the runner settings so we can include them in benchmark reports
// also, toggles tracing on or off
var runnerSettings = new RunnerSettings()
{
ConcurrentModeEnabled = _package.Concurrent,
TracingEnabled = _package.Tracing
};
IBenchmarkOutput output = CreateOutput();
var discovery = new ReflectionDiscovery(output,
DefaultBenchmarkAssertionRunner.Instance, // one day we might be able to pass in custom assertion runners, hence why this is here
runnerSettings);
var result = new TestRunnerResult()
{
AllTestsPassed = true,
FullResults = _results
};
try
{
foreach (var assembly in _package.TestAssemblies)
{
output.WriteLine($"Executing Benchmarks in {assembly}");
var benchmarks = discovery.FindBenchmarks(assembly);
foreach (var benchmark in benchmarks)
{
// verify if the benchmark should be included/excluded from the list of benchmarks to be run
if (_package.ShouldRunBenchmark(benchmark.BenchmarkName))
{
output.StartBenchmark(benchmark.BenchmarkName);
benchmark.Run();
benchmark.Finish();
// if one assert fails, all fail
result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed;
output.FinishBenchmark(benchmark.BenchmarkName);
result.ExecutedTestsCount = result.ExecutedTestsCount + 1;
}
else
{
output.SkipBenchmark(benchmark.BenchmarkName);
result.IgnoredTestsCount = result.IgnoredTestsCount + 1;
}
}
}
}
catch (Exception ex)
{
output.Error(ex, "Error while executing the tests.");
result.AllTestsPassed = false;
}
return(result);
}