/// <summary> /// Executes the tests /// </summary> /// <returns>True if all tests passed.</returns> public TestRunnerResult Execute() { // Perform core / thread optimizations if we're running in single-threaded mode // But not if the user has specified that they're going to be running multi-threaded benchmarks SetProcessPriority(_package.Concurrent); IBenchmarkOutput output = CreateOutput(); var discovery = new ReflectionDiscovery(output); var result = new TestRunnerResult() { AllTestsPassed = true }; try { foreach (var testFile in _package.Files) { var assembly = AssemblyRuntimeLoader.LoadAssembly(testFile); var benchmarks = discovery.FindBenchmarks(assembly); foreach (var benchmark in benchmarks) { // verify if the benchmark should be included/excluded from the list of benchmarks to be run if (_package.ShouldRunBenchmark(benchmark.BenchmarkName)) { output.WriteLine($"------------ STARTING {benchmark.BenchmarkName} ---------- "); benchmark.Run(); benchmark.Finish(); // if one assert fails, all fail result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed; output.WriteLine($"------------ FINISHED {benchmark.BenchmarkName} ---------- "); result.ExecutedTestsCount = result.ExecutedTestsCount + 1; } else { output.WriteLine($"------------ NOTRUN {benchmark.BenchmarkName} ---------- "); result.IgnoredTestsCount = result.IgnoredTestsCount + 1; } } } } catch (Exception ex) { output.Error(ex, "Error while executing the tests."); result.AllTestsPassed = false; } return(result); }
/// <summary> /// Executes the tests /// </summary> /// <returns>True if all tests passed.</returns> public TestRunnerResult Execute() { // Perform core / thread optimizations if we're running in single-threaded mode // But not if the user has specified that they're going to be running multi-threaded benchmarks SetProcessPriority(_package.Concurrent); // pass in the runner settings so we can include them in benchmark reports // also, toggles tracing on or off var runnerSettings = new RunnerSettings() { ConcurrentModeEnabled = _package.Concurrent, TracingEnabled = _package.Tracing }; IBenchmarkOutput output = CreateOutput(); var discovery = new ReflectionDiscovery(output, DefaultBenchmarkAssertionRunner.Instance, // one day we might be able to pass in custom assertion runners, hence why this is here runnerSettings); var result = new TestRunnerResult() { AllTestsPassed = true }; try { foreach (var testFile in _package.Files) { var assembly = AssemblyRuntimeLoader.LoadAssembly(testFile); var benchmarks = discovery.FindBenchmarks(assembly); foreach (var benchmark in benchmarks) { // verify if the benchmark should be included/excluded from the list of benchmarks to be run if (_package.ShouldRunBenchmark(benchmark.BenchmarkName)) { output.StartBenchmark(benchmark.BenchmarkName); benchmark.Run(); benchmark.Finish(); // if one assert fails, all fail result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed; output.FinishBenchmark(benchmark.BenchmarkName); result.ExecutedTestsCount = result.ExecutedTestsCount + 1; } else { output.SkipBenchmark(benchmark.BenchmarkName); result.IgnoredTestsCount = result.IgnoredTestsCount + 1; } } } } catch (Exception ex) { output.Error(ex, "Error while executing the tests."); result.AllTestsPassed = false; } return(result); }