public void ArrayEnumerationIsFaster() { // # Arrange IBenchmarkValidator validator = LatencyValidatorFactory.Builder .IfTreatmentFasterThanBaseline(byAtLeast: 10.Percent(), withConfidenceLevel: 0.99, then: LatencyValidatorBehavior.Pass) .Otherwise(LatencyValidatorBehavior.Fail); var validators = new[] { validator }; // # Act ISpecificBenchmarkRunner runner = benchmarkRunner.ForBenchmarkContainer <ArrayEnumerationIsFaster_Benchmarks>(); // Not strictly necessary // TODO: We should change how RunBenchmark is called to incorporate limits on how much time we are willing to spend //{ // BenchmarkRunEstimate runEstimate = runner.GetRunEstimate(validators); // if (runEstimate.EstimatedTime > TimeSpan.FromMinutes(2)) // { // Assert.Inconclusive("Inconclusive - It would take too long"); // } //} BenchmarkResults benchmarkResults = runner.RunBenchmark(forValidators: validators); BenchmarkAssert.ValidatorsPassed( validators, benchmarkResults, assertFailDelegate: Assert.Fail); }
private static void Main() { // # Arrange IBenchmarkValidator validator = LatencyValidatorFactory.Builder //.IfTreatmentSlowerThanBaseline(withConfidenceLevel: 0.99, then: LatencyValidatorBehavior.Fail) .IfTreatmentFasterThanBaseline(byAtLeast: 0.Percent(), withConfidenceLevel: 0.95, then: LatencyValidatorBehavior.Pass) .Otherwise(LatencyValidatorBehavior.Fail); var validators = new[] { validator }; // # Act ISpecificBenchmarkRunner runnerForString = benchmarkRunner.ForBenchmarkContainer <A>(); // Not strictly necessary //{ // BenchmarkRunEstimate runEstimate = runnerForString.GetRunEstimate(validators); // var alternativeEstimate = benchmarkRunner.GetRunEstimate<string>(validators); // if (runEstimate.EstimatedTime > TimeSpan.FromMinutes(2)) // { // Debug.Fail("Inconclusive - It would take too long"); // } //} BenchmarkResults benchmarkResults = runnerForString.RunBenchmark(validators); BenchmarkAssert.ValidatorsPassed( validators, benchmarkResults, assertFailDelegate: s => Debug.Fail(s)); }
public static BenchmarkResults RunBenchmark( this ISpecificBenchmarkRunner runner, IEnumerable <IBenchmarkValidator> forValidators) { // TODO: Restore this //var estimate = runner.GetRunEstimate(forSampleSizeDeterminers); //return runner.RunBenchmark(estimate.RunParameters); return(runner.RunBenchmark(new BenchmarkRunParameters(TimeSpan.FromSeconds(1)))); }
public static void RunWithValidatorAndAssertPassed( this ISpecificBenchmarkRunner runner, IBenchmarkValidator validator, Action <string> assertFailDelegate) { RunWithValidatorsAndAssertPassed( runner, new[] { validator }, assertFailDelegate); }
public static void RunWithValidatorsAndAssertPassed( this ISpecificBenchmarkRunner runner, IEnumerable <IBenchmarkValidator> validators, Action <string> assertFailDelegate) { // Single enumeration validators = validators.ToList(); var results = runner.RunBenchmark(validators); BenchmarkAssert.ValidatorsPassed( validators, results, assertFailDelegate); }
public void EnumerableEnumerationIsFaster() { // # Arrange ISpecificBenchmarkRunner runner = benchmarkRunner/*Factory? Context? TODO*/ .For( baseline: (EnumerableEnumerationIsFaster_Benchmarks container) => container.ListEnumeration(), treatment: (EnumerableEnumerationIsFaster_Benchmarks container) => container.EnumerableEnumeration()); IBenchmarkValidator validator = LatencyValidatorFactory.Builder // treatment: EnumerableEnumeration <slower than> baseline: ListEnumeration .IfTreatmentSlowerThanBaseline(byAtLeast: 20.Percent(), withConfidenceLevel: 0.95, then: LatencyValidatorBehavior.Pass) .Otherwise(LatencyValidatorBehavior.Fail); // # Act BenchmarkResults benchmarkResults = runner.RunBenchmark( // TODO: Would 'sampleSizeDeterminers' be better? I mean, THAT name is horrible, but more accurate forValidator: validator); // # Assert BenchmarkAssert.ValidatorsPassed( new [] { validator }, benchmarkResults, assertFailDelegate: Assert.Fail); // NOTE - same as above, more succinctly... //DefaultBenchmarkRunner.Instance // .For( // baseline: (EnumerableEnumerationIsFaster_Benchmarks container) => container.ListEnumeration(), // treatment: (EnumerableEnumerationIsFaster_Benchmarks container) => container.EnumerableEnumeration()) // // TODO: Would rather Run() and then Assert(), but both require the arguments right now so wouldn't be able to do that in-line reading like a sentence // // We are trying to represent... // // "For baseline ListEnumeration and treatment EnumerableEnumeration if treatment is slower than baseline with confidence level 0.9999 then pass // // otherwise fail by calling Assert.Fail()" // .RunWithValidatorAndAssertPassed( // LatencyValidatorFactory.Builder // .IfTreatmentSlowerThanBaseline( // byAtLeast: 10.Percent(), // withConfidenceLevel: 0.9999, // then: LatencyValidatorBehavior.Pass) // .Otherwise(LatencyValidatorBehavior.Fail), // assertFailDelegate: Assert.Fail); }
public static BenchmarkResults RunBenchmark( this ISpecificBenchmarkRunner runner, IBenchmarkValidator forValidator) { return(RunBenchmark(runner, new[] { forValidator })); }