public void Setup()
{
var discovery = new ReflectionDiscovery(NoOpBenchmarkOutput.Instance);
var benchmarkData = ReflectionDiscovery.CreateBenchmarksForClass(typeof (MemoryAllocationSpec)).First();
var settings = discovery.CreateSettingsForBenchmark(benchmarkData);
var invoker = ReflectionDiscovery.CreateInvokerForBenchmark(benchmarkData);
_testableBenchmark = new Benchmark(settings, invoker, BenchmarkOutput);
}
public void BugFix153IsFixed()
{
var o = new ActionBenchmarkOutput(benchmarkAction: r =>
{
var name = new CounterMetricName(CounterThroughputBenchmark.CounterName);
var count = r.Data.StatsByMetric[name];
Assert.True(count.PerSecondStats.Average > 0.0);
});
var d = new ReflectionDiscovery(o, DefaultBenchmarkAssertionRunner.Instance, new RunnerSettings());
var benchmarks = d.FindBenchmarks(typeof(CounterThroughputBenchmark));
foreach (var b in benchmarks)
{
b.Run();
b.Finish();
Assert.True(b.AllAssertsPassed);
}
}
public void Should_write_trace_messages_in_user_spec_to_output_when_runner_has_enabled_tracing()
{
var methodOutput = new List<string>();
var discovery = new ReflectionDiscovery(new ActionBenchmarkOutput(writeLineAction: str =>
{
methodOutput.Add(str);
}), DefaultBenchmarkAssertionRunner.Instance, new RunnerSettings() { TracingEnabled = true});
var benchmarks = discovery.FindBenchmarks(typeof (TracingBenchmark)).ToList();
Assert.Equal(1, benchmarks.Count); // sanity check
foreach(var benchmark in benchmarks)
benchmark.Run();
var setupTraces = methodOutput.Count(x => x.Contains(SetupTrace));
var runTraces = methodOutput.Count(x => x.Contains(RunTrace));
var cleanupTraces = methodOutput.Count(x => x.Contains(CleanupTrace));
Assert.Equal(ExpectedTraces, setupTraces);
Assert.Equal(ExpectedTraces, runTraces);
Assert.Equal(ExpectedTraces, cleanupTraces);
}
/// <summary>
/// Executes the tests
/// </summary>
/// <returns>True if all tests passed.</returns>
public TestRunnerResult Execute()
{
// Perform core / thread optimizations if we're running in single-threaded mode
// But not if the user has specified that they're going to be running multi-threaded benchmarks
SetProcessPriority(_package.Concurrent);
// pass in the runner settings so we can include them in benchmark reports
// also, toggles tracing on or off
var runnerSettings = new RunnerSettings()
{
ConcurrentModeEnabled = _package.Concurrent,
TracingEnabled = _package.Tracing
};
IBenchmarkOutput output = CreateOutput();
var discovery = new ReflectionDiscovery(output,
DefaultBenchmarkAssertionRunner.Instance, // one day we might be able to pass in custom assertion runners, hence why this is here
runnerSettings);
var result = new TestRunnerResult()
{
AllTestsPassed = true
};
try
{
foreach (var testFile in _package.Files)
{
var assembly = AssemblyRuntimeLoader.LoadAssembly(testFile);
var benchmarks = discovery.FindBenchmarks(assembly);
foreach (var benchmark in benchmarks)
{
// verify if the benchmark should be included/excluded from the list of benchmarks to be run
if (_package.ShouldRunBenchmark(benchmark.BenchmarkName))
{
output.StartBenchmark(benchmark.BenchmarkName);
benchmark.Run();
benchmark.Finish();
// if one assert fails, all fail
result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed;
output.FinishBenchmark(benchmark.BenchmarkName);
result.ExecutedTestsCount = result.ExecutedTestsCount + 1;
}
else
{
output.SkipBenchmark(benchmark.BenchmarkName);
result.IgnoredTestsCount = result.IgnoredTestsCount + 1;
}
}
}
}
catch (Exception ex)
{
output.Error(ex, "Error while executing the tests.");
result.AllTestsPassed = false;
}
return result;
}
public void ShouldProduceBenchmarkSettings_Complex()
{
var discovery = new ReflectionDiscovery(NoOpBenchmarkOutput.Instance);
var benchmarkMetaData = ReflectionDiscovery.CreateBenchmarksForClass(ComplexBenchmarkTypeInfo);
var benchmarkSettings = discovery.CreateSettingsForBenchmark(benchmarkMetaData.First());
Assert.Equal(TestMode.Test, benchmarkSettings.TestMode);
Assert.Equal(PerfBenchmarkAttribute.DefaultRunType, benchmarkSettings.RunMode);
Assert.Equal(0, benchmarkSettings.Measurements.Count(x => x is GcBenchmarkSetting));
Assert.Equal(2, benchmarkSettings.Measurements.Count(x => x is MemoryBenchmarkSetting));
Assert.Equal(1, benchmarkSettings.DistinctMeasurements.Count(x => x is MemoryBenchmarkSetting));
Assert.Equal(0, benchmarkSettings.CounterMeasurements.Count());
}
