예제 #1
0
 /// <summary>
 /// Initializes a new instance of the test runner.
 /// </summary>
 /// <param name="package">The test package to be executed</param>
 public TestRunner(TestPackage package)
 {
     _package          = package;
     _resultsCollector = new ActionBenchmarkOutput(benchmarkAction: f => { _results.Add(f); });
 }
예제 #2
0
        /// <summary>
        /// Executes the tests
        /// </summary>
        /// <returns>True if all tests passed.</returns>
        public TestRunnerResult Execute()
        {
            // Perform core / thread optimizations if we're running in single-threaded mode
            // But not if the user has specified that they're going to be running multi-threaded benchmarks
            SetProcessPriority(_package.Concurrent);

            // pass in the runner settings so we can include them in benchmark reports
            // also, toggles tracing on or off
            var runnerSettings = new RunnerSettings()
            {
                ConcurrentModeEnabled = _package.Concurrent,
                TracingEnabled        = _package.Tracing
            };

            IBenchmarkOutput output = CreateOutput();


            var discovery = new ReflectionDiscovery(output,
                                                    DefaultBenchmarkAssertionRunner.Instance, // one day we might be able to pass in custom assertion runners, hence why this is here
                                                    runnerSettings);
            var result = new TestRunnerResult()
            {
                AllTestsPassed = true,
                FullResults    = _results
            };

            try
            {
                foreach (var assembly in _package.TestAssemblies)
                {
                    output.WriteLine($"Executing Benchmarks in {assembly}");
                    var benchmarks = discovery.FindBenchmarks(assembly);

                    foreach (var benchmark in benchmarks)
                    {
                        // verify if the benchmark should be included/excluded from the list of benchmarks to be run
                        if (_package.ShouldRunBenchmark(benchmark.BenchmarkName))
                        {
                            output.StartBenchmark(benchmark.BenchmarkName);
                            benchmark.Run();
                            benchmark.Finish();

                            // if one assert fails, all fail
                            result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed;
                            output.FinishBenchmark(benchmark.BenchmarkName);
                            result.ExecutedTestsCount = result.ExecutedTestsCount + 1;
                        }
                        else
                        {
                            output.SkipBenchmark(benchmark.BenchmarkName);
                            result.IgnoredTestsCount = result.IgnoredTestsCount + 1;
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                output.Error(ex, "Error while executing the tests.");
                result.AllTestsPassed = false;
            }

            return(result);
        }