public IEnumerable <ValidationResult> GetValidationResults(BenchmarkResults results)
{
foreach (var result in results.ResultsByCase)
{
yield return(GetValidationResult(result.Key, result.Value));
}
}
public static void ValidatorsPassed(
IEnumerable <IBenchmarkValidator> validators,
BenchmarkResults results,
// Delegate to support multiple test frameworks
Action <string> assertFailDelegate)
{
// Get results
IReadOnlyList <ValidationResult> allResults = validators
.SelectMany(validator => validator.GetValidationResults(results))
.ToArray();
// Output all results for debugging
Console.WriteLine(GetOutputMessage(allResults));
// Fail if there are any violations
IReadOnlyList <ValidationResult> failedResults = allResults
.Where(validationResult => validationResult.IsViolation)
.ToArray();
if (failedResults.Any())
{
var failedMessage = GetOutputMessage(failedResults);
assertFailDelegate(failedMessage);
}
}
private static void Main()
{
// # Arrange
IBenchmarkValidator validator = LatencyValidatorFactory.Builder
//.IfTreatmentSlowerThanBaseline(withConfidenceLevel: 0.99, then: LatencyValidatorBehavior.Fail)
.IfTreatmentFasterThanBaseline(byAtLeast: 0.Percent(), withConfidenceLevel: 0.95, then: LatencyValidatorBehavior.Pass)
.Otherwise(LatencyValidatorBehavior.Fail);
var validators = new[] { validator };
// # Act
ISpecificBenchmarkRunner runnerForString = benchmarkRunner.ForBenchmarkContainer <A>();
// Not strictly necessary
//{
// BenchmarkRunEstimate runEstimate = runnerForString.GetRunEstimate(validators);
// var alternativeEstimate = benchmarkRunner.GetRunEstimate<string>(validators);
// if (runEstimate.EstimatedTime > TimeSpan.FromMinutes(2))
// {
// Debug.Fail("Inconclusive - It would take too long");
// }
//}
BenchmarkResults benchmarkResults = runnerForString.RunBenchmark(validators);
BenchmarkAssert.ValidatorsPassed(
validators,
benchmarkResults,
assertFailDelegate: s => Debug.Fail(s));
}
/// <summary>
/// Returns validator as it must be the terminal method
/// aka: Build()
/// </summary>
/// <param name="fallbackBehavior"></param>
/// <returns></returns>
public IBenchmarkValidator Otherwise(LatencyValidatorBehavior fallbackBehavior)
{
var finalSteps = RemoveRedundantSteps(this.steps, fallbackBehavior);
var totalNumberOfTests = finalSteps.Count;
var validatorsAndBehaviors = finalSteps
.Select(step =>
{
var alpha = 1 - step.ConfidenceLevel;
var modifiedAlpha = alpha / totalNumberOfTests;
TwoSampleHypothesis alternate;
switch (step.StepType)
{
case BuilderStepType.IfFasterThan:
alternate = TwoSampleHypothesis.FirstValueIsGreaterThanSecond;
break;
case BuilderStepType.IfSlowerThan:
alternate = TwoSampleHypothesis.FirstValueIsSmallerThanSecond;
break;
default:
throw new ArgumentOutOfRangeException();
}
BothLatencyValidator validator;
if (step.ByAtLeastTimeInterval != null)
{
validator = new BothLatencyValidator(
// TODO: P3 - Pass in things like the detectable difference and power?
modifiedAlpha, alternate,
byAtLeast: step.ByAtLeastTimeInterval.Value);
}
else if (step.ByAtLeastPercent != null)
{
validator = new BothLatencyValidator(
// TODO: P3 - Pass in things like the detectable difference and power?
modifiedAlpha, alternate,
byAtLeast: step.ByAtLeastPercent.Value);
}
else
{
throw new Exception();
}
return(new
{
validator = (IBenchmarkValidator)validator,
sampleSizeDeterminer = (ISampleSizeDeterminer)validator,
ifViolation = step.Behavior
});
})
.ToArray();
// TODO: Too much logic here
return(new DelegateBenchmarkValidator(
basedOnPreliminaryResults =>
{
var samplesRequirements = validatorsAndBehaviors
.Select(valWithBehavior => valWithBehavior.sampleSizeDeterminer)
.Select(sampleSizeDeterminer => sampleSizeDeterminer.GetSampleSizeRequirement(basedOnPreliminaryResults))
.ToArray();
int samplesForBaseline = samplesRequirements
.Max(size => size.SamplesForBaseline);
int samplesForTreatment = samplesRequirements
.Max(size => size.SamplesForTreatment);
return new SamplesRequirement(
samplesForBaseline,
samplesForTreatment);
},
results =>
{
var toReturn = new List <ValidationResult>();
foreach (var resultAndCase in results.ResultsByCase)
{
var singleItemBenchmarkResult = new BenchmarkResults(
new Dictionary <ParameterInstances, BenchmarkResults.BeforeAndAfter>
{
[resultAndCase.Key] = resultAndCase.Value,
});
StringBuilder sb = new StringBuilder();
ValidationResult validationResult = null;
foreach (var valWithBehavior in validatorsAndBehaviors)
{
var result = valWithBehavior.validator.GetValidationResults(singleItemBenchmarkResult).Single();
sb.AppendLine($"Condition {result.Validator} was {result.IsViolation} match - {result.Message}");
if (result.IsViolation)
{
validationResult = new ValidationResult(
resultAndCase.Key,
result.Validator,
sb.ToString(),
// TODO: P1 - This doesn't handle inconclusive - there's no way for us to do that?
isViolation: valWithBehavior.ifViolation == LatencyValidatorBehavior.Fail);
// Stop running validators for this collection
break;
}
}
sb.AppendLine($"No condition was satisfied, so using the fallback {fallbackBehavior}");
if (validationResult == null)
{
// use the fallback
validationResult = new ValidationResult(
resultAndCase.Key,
null, // Need the 'validator' that created this
sb.ToString(),
isViolation: fallbackBehavior == LatencyValidatorBehavior.Fail);
}
toReturn.Add(validationResult);
}
return toReturn;
}));
}
IEnumerable <ValidationResult> IBenchmarkValidator.GetValidationResults(BenchmarkResults results)
{
return(this.getValidationResults(results));
}
