public BuilderStep(BuilderStepType stepType, double confidenceLevel, LatencyValidatorBehavior behavior, TimeInterval byAtLeast)
{
this.ByAtLeastTimeInterval = byAtLeast;
this.StepType = stepType;
this.ConfidenceLevel = confidenceLevel;
this.Behavior = behavior;
}
public BuilderStep(BuilderStepType stepType, double confidenceLevel, LatencyValidatorBehavior behavior, Percent byAtLeast)
{
this.ByAtLeastPercent = byAtLeast;
this.StepType = stepType;
this.ConfidenceLevel = confidenceLevel;
this.Behavior = behavior;
}
public LatencyValidatorBuilder IfTreatmentFasterThanBaseline(
TimeInterval byAtLeast,
double withConfidenceLevel, LatencyValidatorBehavior then)
{
return(new LatencyValidatorBuilder(
this.steps
.Append(new BuilderStep(BuilderStepType.IfFasterThan, withConfidenceLevel, then, byAtLeast))
.ToList()));
}
public LatencyValidatorBuilder IfTreatmentSlowerThanBaseline(
// TODO: Convert all these to OneOf when we have internet
TimeInterval byAtLeast,
double withConfidenceLevel,
LatencyValidatorBehavior then)
{
return(new LatencyValidatorBuilder(
this.steps
.Append(new BuilderStep(BuilderStepType.IfSlowerThan, withConfidenceLevel, then, byAtLeast))
.ToList()));
}
private static IReadOnlyList <BuilderStep> RemoveRedundantSteps(
IReadOnlyList <BuilderStep> readOnlyList,
LatencyValidatorBehavior fallbackBehavior)
{
List <BuilderStep> steps = readOnlyList.ToList();
// Remove any from the end that match the fallback
while (steps.Any() && steps.Last().Behavior == fallbackBehavior)
{
steps.RemoveAt(steps.Count - 1);
}
return(steps);
}
/// <summary>
/// Returns validator as it must be the terminal method
/// aka: Build()
/// </summary>
/// <param name="fallbackBehavior"></param>
/// <returns></returns>
public IBenchmarkValidator Otherwise(LatencyValidatorBehavior fallbackBehavior)
{
var finalSteps = RemoveRedundantSteps(this.steps, fallbackBehavior);
var totalNumberOfTests = finalSteps.Count;
var validatorsAndBehaviors = finalSteps
.Select(step =>
{
var alpha = 1 - step.ConfidenceLevel;
var modifiedAlpha = alpha / totalNumberOfTests;
TwoSampleHypothesis alternate;
switch (step.StepType)
{
case BuilderStepType.IfFasterThan:
alternate = TwoSampleHypothesis.FirstValueIsGreaterThanSecond;
break;
case BuilderStepType.IfSlowerThan:
alternate = TwoSampleHypothesis.FirstValueIsSmallerThanSecond;
break;
default:
throw new ArgumentOutOfRangeException();
}
BothLatencyValidator validator;
if (step.ByAtLeastTimeInterval != null)
{
validator = new BothLatencyValidator(
// TODO: P3 - Pass in things like the detectable difference and power?
modifiedAlpha, alternate,
byAtLeast: step.ByAtLeastTimeInterval.Value);
}
else if (step.ByAtLeastPercent != null)
{
validator = new BothLatencyValidator(
// TODO: P3 - Pass in things like the detectable difference and power?
modifiedAlpha, alternate,
byAtLeast: step.ByAtLeastPercent.Value);
}
else
{
throw new Exception();
}
return(new
{
validator = (IBenchmarkValidator)validator,
sampleSizeDeterminer = (ISampleSizeDeterminer)validator,
ifViolation = step.Behavior
});
})
.ToArray();
// TODO: Too much logic here
return(new DelegateBenchmarkValidator(
basedOnPreliminaryResults =>
{
var samplesRequirements = validatorsAndBehaviors
.Select(valWithBehavior => valWithBehavior.sampleSizeDeterminer)
.Select(sampleSizeDeterminer => sampleSizeDeterminer.GetSampleSizeRequirement(basedOnPreliminaryResults))
.ToArray();
int samplesForBaseline = samplesRequirements
.Max(size => size.SamplesForBaseline);
int samplesForTreatment = samplesRequirements
.Max(size => size.SamplesForTreatment);
return new SamplesRequirement(
samplesForBaseline,
samplesForTreatment);
},
results =>
{
var toReturn = new List <ValidationResult>();
foreach (var resultAndCase in results.ResultsByCase)
{
var singleItemBenchmarkResult = new BenchmarkResults(
new Dictionary <ParameterInstances, BenchmarkResults.BeforeAndAfter>
{
[resultAndCase.Key] = resultAndCase.Value,
});
StringBuilder sb = new StringBuilder();
ValidationResult validationResult = null;
foreach (var valWithBehavior in validatorsAndBehaviors)
{
var result = valWithBehavior.validator.GetValidationResults(singleItemBenchmarkResult).Single();
sb.AppendLine($"Condition {result.Validator} was {result.IsViolation} match - {result.Message}");
if (result.IsViolation)
{
validationResult = new ValidationResult(
resultAndCase.Key,
result.Validator,
sb.ToString(),
// TODO: P1 - This doesn't handle inconclusive - there's no way for us to do that?
isViolation: valWithBehavior.ifViolation == LatencyValidatorBehavior.Fail);
// Stop running validators for this collection
break;
}
}
sb.AppendLine($"No condition was satisfied, so using the fallback {fallbackBehavior}");
if (validationResult == null)
{
// use the fallback
validationResult = new ValidationResult(
resultAndCase.Key,
null, // Need the 'validator' that created this
sb.ToString(),
isViolation: fallbackBehavior == LatencyValidatorBehavior.Fail);
}
toReturn.Add(validationResult);
}
return toReturn;
}));
}
