public BenchmarkResults RunBenchmark(BenchmarkRunParameters runParameters)
{
var genericMethod = typeof(IBenchmarkRunner)
.GetMethod(nameof(IBenchmarkRunner.RunBenchmark));
var method = genericMethod
.MakeGenericMethod(this.BenchmarkClass);
var result = method.Invoke(this.runner, parameters: new object[] { runParameters });
var castResult = (BenchmarkResults)result;
return(castResult);
}
public BenchmarkResults RunBenchmark <TBenchmarkContainer>(BenchmarkRunParameters runParameters)
{
var config = new Config(
runParameters.DesiredMaxLatency,
this.jobMutator);
// TODO: P3 - Validate return values to catch invalid usage (e.g. Before throws and After returns - invalid Benchmark comparison because not doing the same thing)
var reports = BenchmarkRunner.Run <TBenchmarkContainer>(config)
.Reports;
var parameterInstancesComparer = ParameterInstancesComparer.Default;
var reportsByArgs = reports
.GroupBy(
report => report.BenchmarkCase.Parameters,
parameterInstancesComparer);
IDictionary <ParameterInstances, BenchmarkResults.BeforeAndAfter> beforeAndAfters =
new Dictionary <ParameterInstances, BenchmarkResults.BeforeAndAfter>(parameterInstancesComparer);
foreach (var reportForArgs in reportsByArgs)
{
if (reportForArgs.Count() != 2 ||
reportForArgs.Count(report => report.BenchmarkCase.IsBaseline()) != 1)
{
throw new InvalidOperationException("Expected exactly 1 baseline and 1 treatment");
}
var args = reportForArgs.Key;
var baseline = reportForArgs.Single(report => report.BenchmarkCase.IsBaseline());
var treatment = reportForArgs.Single(report => !report.BenchmarkCase.IsBaseline());
beforeAndAfters[args] = new BenchmarkResults.BeforeAndAfter(
baseline,
treatment);
}
return(new BenchmarkResults(beforeAndAfters));
}
public BenchmarkRunEstimate GetRunEstimate <TBenchmarkContainer>(
IEnumerable <ISampleSizeDeterminer> sampleSizeDeterminers)
{
// TODO: Point of this class was to cache - needs to cache :)
// TODO: P2 - Get the run time for the preliminary FROM a simple invocation that sees how long the methods take.
// E.G. Do not limit to 10s if the method itself takes 30s
var preliminaryRunParameters = new BenchmarkRunParameters(desiredMaxLatency: TimeSpan.FromSeconds(10));
var preliminaryRunResults = this.RunBenchmark <TBenchmarkContainer>(preliminaryRunParameters);
IReadOnlyDictionary <ISampleSizeDeterminer, TimeSpan> estimatedTimeBySource =
sampleSizeDeterminers
.ToDictionary(
sampleSizeDeterminer => sampleSizeDeterminer,
sampleSizeDeterminer =>
{
// The time taken for the validator is defined as
// Sum(
// each: parameterCase
// case ->
// iterationCount = max(treatmentIterationCountRequired, baselineIterationCountRequired)
// iterationCount * treatmentTime + iterationCount * baselineTime
// )
TimeSpan estimatedDuration = TimeSpan.Zero;
foreach (var resultAndCase in preliminaryRunResults.ResultsByCase)
{
var benchmarkCase = resultAndCase.Key;
var baselineReport = resultAndCase.Value.Baseline;
var treatmentReport = resultAndCase.Value.Treatment;
var sampleSizeRequirementForBaseline =
sampleSizeDeterminer.GetSampleSizeRequirement(resultAndCase.Value).SamplesForBaseline;
var sampleSizeRequirementForTreatment =
sampleSizeDeterminer.GetSampleSizeRequirement(resultAndCase.Value).SamplesForTreatment;
// As of now, we run both for the same amount of samples
sampleSizeRequirementForBaseline =
Math.Max((int)sampleSizeRequirementForBaseline, (int)sampleSizeRequirementForTreatment);
sampleSizeRequirementForTreatment =
Math.Max((int)sampleSizeRequirementForBaseline, (int)sampleSizeRequirementForTreatment);
var nanosecondsPerSampleInBaseline = baselineReport
.GetResultRuns()
.Average(workloadMeasurement => workloadMeasurement.Nanoseconds);
var timePerSampleInBaseline =
TimeSpan.FromTicks((long)(nanosecondsPerSampleInBaseline / 100));
var nanosecondsPerSampleInTreatment = treatmentReport
.GetResultRuns()
.Average(workloadMeasurement => workloadMeasurement.Nanoseconds);
var timePerSampleInTreatment =
TimeSpan.FromTicks((long)(nanosecondsPerSampleInTreatment / 100));
var totalTimeForBaseline = TimeSpan.FromMilliseconds(timePerSampleInBaseline.TotalMilliseconds * sampleSizeRequirementForBaseline);
var totalTimeForTreatment = TimeSpan.FromMilliseconds(timePerSampleInTreatment.TotalMilliseconds * sampleSizeRequirementForTreatment);
var totalTimeForCase = totalTimeForBaseline + totalTimeForTreatment;
estimatedDuration += totalTimeForCase;
}
return(estimatedDuration);
});
var maxTime = estimatedTimeBySource.Values.Max();
// TODO: P2 - Need to encapsulate the information about each parameters and how many ITERATIONS they should run for from above
BenchmarkRunParameters runParameters = new BenchmarkRunParameters(maxTime);
return(new BenchmarkRunEstimate(
maxTime,
runParameters,
estimatedTimeBySource));
}
public BenchmarkRunEstimate(TimeSpan estimatedTime, BenchmarkRunParameters runParameters, IReadOnlyDictionary <ISampleSizeDeterminer, TimeSpan> estimatedTimeBySource)
{
this.EstimatedTime = estimatedTime;
this.RunParameters = runParameters;
this.EstimatedTimeBySource = estimatedTimeBySource;
}
