public IEnumerable <ValidationResult> GetValidationResults(BenchmarkResults results) { foreach (var result in results.ResultsByCase) { yield return(GetValidationResult(result.Key, result.Value)); } }
public static void ValidatorsPassed( IEnumerable <IBenchmarkValidator> validators, BenchmarkResults results, // Delegate to support multiple test frameworks Action <string> assertFailDelegate) { // Get results IReadOnlyList <ValidationResult> allResults = validators .SelectMany(validator => validator.GetValidationResults(results)) .ToArray(); // Output all results for debugging Console.WriteLine(GetOutputMessage(allResults)); // Fail if there are any violations IReadOnlyList <ValidationResult> failedResults = allResults .Where(validationResult => validationResult.IsViolation) .ToArray(); if (failedResults.Any()) { var failedMessage = GetOutputMessage(failedResults); assertFailDelegate(failedMessage); } }
private static void Main() { // # Arrange IBenchmarkValidator validator = LatencyValidatorFactory.Builder //.IfTreatmentSlowerThanBaseline(withConfidenceLevel: 0.99, then: LatencyValidatorBehavior.Fail) .IfTreatmentFasterThanBaseline(byAtLeast: 0.Percent(), withConfidenceLevel: 0.95, then: LatencyValidatorBehavior.Pass) .Otherwise(LatencyValidatorBehavior.Fail); var validators = new[] { validator }; // # Act ISpecificBenchmarkRunner runnerForString = benchmarkRunner.ForBenchmarkContainer <A>(); // Not strictly necessary //{ // BenchmarkRunEstimate runEstimate = runnerForString.GetRunEstimate(validators); // var alternativeEstimate = benchmarkRunner.GetRunEstimate<string>(validators); // if (runEstimate.EstimatedTime > TimeSpan.FromMinutes(2)) // { // Debug.Fail("Inconclusive - It would take too long"); // } //} BenchmarkResults benchmarkResults = runnerForString.RunBenchmark(validators); BenchmarkAssert.ValidatorsPassed( validators, benchmarkResults, assertFailDelegate: s => Debug.Fail(s)); }
/// <summary> /// Returns validator as it must be the terminal method /// aka: Build() /// </summary> /// <param name="fallbackBehavior"></param> /// <returns></returns> public IBenchmarkValidator Otherwise(LatencyValidatorBehavior fallbackBehavior) { var finalSteps = RemoveRedundantSteps(this.steps, fallbackBehavior); var totalNumberOfTests = finalSteps.Count; var validatorsAndBehaviors = finalSteps .Select(step => { var alpha = 1 - step.ConfidenceLevel; var modifiedAlpha = alpha / totalNumberOfTests; TwoSampleHypothesis alternate; switch (step.StepType) { case BuilderStepType.IfFasterThan: alternate = TwoSampleHypothesis.FirstValueIsGreaterThanSecond; break; case BuilderStepType.IfSlowerThan: alternate = TwoSampleHypothesis.FirstValueIsSmallerThanSecond; break; default: throw new ArgumentOutOfRangeException(); } BothLatencyValidator validator; if (step.ByAtLeastTimeInterval != null) { validator = new BothLatencyValidator( // TODO: P3 - Pass in things like the detectable difference and power? modifiedAlpha, alternate, byAtLeast: step.ByAtLeastTimeInterval.Value); } else if (step.ByAtLeastPercent != null) { validator = new BothLatencyValidator( // TODO: P3 - Pass in things like the detectable difference and power? modifiedAlpha, alternate, byAtLeast: step.ByAtLeastPercent.Value); } else { throw new Exception(); } return(new { validator = (IBenchmarkValidator)validator, sampleSizeDeterminer = (ISampleSizeDeterminer)validator, ifViolation = step.Behavior }); }) .ToArray(); // TODO: Too much logic here return(new DelegateBenchmarkValidator( basedOnPreliminaryResults => { var samplesRequirements = validatorsAndBehaviors .Select(valWithBehavior => valWithBehavior.sampleSizeDeterminer) .Select(sampleSizeDeterminer => sampleSizeDeterminer.GetSampleSizeRequirement(basedOnPreliminaryResults)) .ToArray(); int samplesForBaseline = samplesRequirements .Max(size => size.SamplesForBaseline); int samplesForTreatment = samplesRequirements .Max(size => size.SamplesForTreatment); return new SamplesRequirement( samplesForBaseline, samplesForTreatment); }, results => { var toReturn = new List <ValidationResult>(); foreach (var resultAndCase in results.ResultsByCase) { var singleItemBenchmarkResult = new BenchmarkResults( new Dictionary <ParameterInstances, BenchmarkResults.BeforeAndAfter> { [resultAndCase.Key] = resultAndCase.Value, }); StringBuilder sb = new StringBuilder(); ValidationResult validationResult = null; foreach (var valWithBehavior in validatorsAndBehaviors) { var result = valWithBehavior.validator.GetValidationResults(singleItemBenchmarkResult).Single(); sb.AppendLine($"Condition {result.Validator} was {result.IsViolation} match - {result.Message}"); if (result.IsViolation) { validationResult = new ValidationResult( resultAndCase.Key, result.Validator, sb.ToString(), // TODO: P1 - This doesn't handle inconclusive - there's no way for us to do that? isViolation: valWithBehavior.ifViolation == LatencyValidatorBehavior.Fail); // Stop running validators for this collection break; } } sb.AppendLine($"No condition was satisfied, so using the fallback {fallbackBehavior}"); if (validationResult == null) { // use the fallback validationResult = new ValidationResult( resultAndCase.Key, null, // Need the 'validator' that created this sb.ToString(), isViolation: fallbackBehavior == LatencyValidatorBehavior.Fail); } toReturn.Add(validationResult); } return toReturn; })); }
IEnumerable <ValidationResult> IBenchmarkValidator.GetValidationResults(BenchmarkResults results) { return(this.getValidationResults(results)); }