public void ArrayEnumerationIsFaster()
{
// # Arrange
IBenchmarkValidator validator = LatencyValidatorFactory.Builder
.IfTreatmentFasterThanBaseline(byAtLeast: 10.Percent(), withConfidenceLevel: 0.99, then: LatencyValidatorBehavior.Pass)
.Otherwise(LatencyValidatorBehavior.Fail);
var validators = new[] { validator };
// # Act
ISpecificBenchmarkRunner runner = benchmarkRunner.ForBenchmarkContainer <ArrayEnumerationIsFaster_Benchmarks>();
// Not strictly necessary
// TODO: We should change how RunBenchmark is called to incorporate limits on how much time we are willing to spend
//{
// BenchmarkRunEstimate runEstimate = runner.GetRunEstimate(validators);
// if (runEstimate.EstimatedTime > TimeSpan.FromMinutes(2))
// {
// Assert.Inconclusive("Inconclusive - It would take too long");
// }
//}
BenchmarkResults benchmarkResults = runner.RunBenchmark(forValidators: validators);
BenchmarkAssert.ValidatorsPassed(
validators,
benchmarkResults,
assertFailDelegate: Assert.Fail);
}
private static void Main()
{
// # Arrange
IBenchmarkValidator validator = LatencyValidatorFactory.Builder
//.IfTreatmentSlowerThanBaseline(withConfidenceLevel: 0.99, then: LatencyValidatorBehavior.Fail)
.IfTreatmentFasterThanBaseline(byAtLeast: 0.Percent(), withConfidenceLevel: 0.95, then: LatencyValidatorBehavior.Pass)
.Otherwise(LatencyValidatorBehavior.Fail);
var validators = new[] { validator };
// # Act
ISpecificBenchmarkRunner runnerForString = benchmarkRunner.ForBenchmarkContainer <A>();
// Not strictly necessary
//{
// BenchmarkRunEstimate runEstimate = runnerForString.GetRunEstimate(validators);
// var alternativeEstimate = benchmarkRunner.GetRunEstimate<string>(validators);
// if (runEstimate.EstimatedTime > TimeSpan.FromMinutes(2))
// {
// Debug.Fail("Inconclusive - It would take too long");
// }
//}
BenchmarkResults benchmarkResults = runnerForString.RunBenchmark(validators);
BenchmarkAssert.ValidatorsPassed(
validators,
benchmarkResults,
assertFailDelegate: s => Debug.Fail(s));
}
public static BenchmarkResults RunBenchmark(
this ISpecificBenchmarkRunner runner,
IEnumerable <IBenchmarkValidator> forValidators)
{
// TODO: Restore this
//var estimate = runner.GetRunEstimate(forSampleSizeDeterminers);
//return runner.RunBenchmark(estimate.RunParameters);
return(runner.RunBenchmark(new BenchmarkRunParameters(TimeSpan.FromSeconds(1))));
}
public static void RunWithValidatorAndAssertPassed(
this ISpecificBenchmarkRunner runner,
IBenchmarkValidator validator,
Action <string> assertFailDelegate)
{
RunWithValidatorsAndAssertPassed(
runner,
new[] { validator },
assertFailDelegate);
}
public static void RunWithValidatorsAndAssertPassed(
this ISpecificBenchmarkRunner runner,
IEnumerable <IBenchmarkValidator> validators,
Action <string> assertFailDelegate)
{
// Single enumeration
validators = validators.ToList();
var results = runner.RunBenchmark(validators);
BenchmarkAssert.ValidatorsPassed(
validators,
results,
assertFailDelegate);
}
public void EnumerableEnumerationIsFaster()
{
// # Arrange
ISpecificBenchmarkRunner runner = benchmarkRunner/*Factory? Context? TODO*/
.For(
baseline: (EnumerableEnumerationIsFaster_Benchmarks container) => container.ListEnumeration(),
treatment: (EnumerableEnumerationIsFaster_Benchmarks container) => container.EnumerableEnumeration());
IBenchmarkValidator validator = LatencyValidatorFactory.Builder
// treatment: EnumerableEnumeration <slower than> baseline: ListEnumeration
.IfTreatmentSlowerThanBaseline(byAtLeast: 20.Percent(), withConfidenceLevel: 0.95, then: LatencyValidatorBehavior.Pass)
.Otherwise(LatencyValidatorBehavior.Fail);
// # Act
BenchmarkResults benchmarkResults = runner.RunBenchmark(
// TODO: Would 'sampleSizeDeterminers' be better? I mean, THAT name is horrible, but more accurate
forValidator: validator);
// # Assert
BenchmarkAssert.ValidatorsPassed(
new [] { validator },
benchmarkResults,
assertFailDelegate: Assert.Fail);
// NOTE - same as above, more succinctly...
//DefaultBenchmarkRunner.Instance
// .For(
// baseline: (EnumerableEnumerationIsFaster_Benchmarks container) => container.ListEnumeration(),
// treatment: (EnumerableEnumerationIsFaster_Benchmarks container) => container.EnumerableEnumeration())
// // TODO: Would rather Run() and then Assert(), but both require the arguments right now so wouldn't be able to do that in-line reading like a sentence
// // We are trying to represent...
// // "For baseline ListEnumeration and treatment EnumerableEnumeration if treatment is slower than baseline with confidence level 0.9999 then pass
// // otherwise fail by calling Assert.Fail()"
// .RunWithValidatorAndAssertPassed(
// LatencyValidatorFactory.Builder
// .IfTreatmentSlowerThanBaseline(
// byAtLeast: 10.Percent(),
// withConfidenceLevel: 0.9999,
// then: LatencyValidatorBehavior.Pass)
// .Otherwise(LatencyValidatorBehavior.Fail),
// assertFailDelegate: Assert.Fail);
}
public static BenchmarkResults RunBenchmark(
this ISpecificBenchmarkRunner runner,
IBenchmarkValidator forValidator)
{
return(RunBenchmark(runner, new[] { forValidator }));
}
