public BenchmarkResults RunBenchmark(BenchmarkRunParameters runParameters) { var genericMethod = typeof(IBenchmarkRunner) .GetMethod(nameof(IBenchmarkRunner.RunBenchmark)); var method = genericMethod .MakeGenericMethod(this.BenchmarkClass); var result = method.Invoke(this.runner, parameters: new object[] { runParameters }); var castResult = (BenchmarkResults)result; return(castResult); }
public BenchmarkResults RunBenchmark <TBenchmarkContainer>(BenchmarkRunParameters runParameters) { var config = new Config( runParameters.DesiredMaxLatency, this.jobMutator); // TODO: P3 - Validate return values to catch invalid usage (e.g. Before throws and After returns - invalid Benchmark comparison because not doing the same thing) var reports = BenchmarkRunner.Run <TBenchmarkContainer>(config) .Reports; var parameterInstancesComparer = ParameterInstancesComparer.Default; var reportsByArgs = reports .GroupBy( report => report.BenchmarkCase.Parameters, parameterInstancesComparer); IDictionary <ParameterInstances, BenchmarkResults.BeforeAndAfter> beforeAndAfters = new Dictionary <ParameterInstances, BenchmarkResults.BeforeAndAfter>(parameterInstancesComparer); foreach (var reportForArgs in reportsByArgs) { if (reportForArgs.Count() != 2 || reportForArgs.Count(report => report.BenchmarkCase.IsBaseline()) != 1) { throw new InvalidOperationException("Expected exactly 1 baseline and 1 treatment"); } var args = reportForArgs.Key; var baseline = reportForArgs.Single(report => report.BenchmarkCase.IsBaseline()); var treatment = reportForArgs.Single(report => !report.BenchmarkCase.IsBaseline()); beforeAndAfters[args] = new BenchmarkResults.BeforeAndAfter( baseline, treatment); } return(new BenchmarkResults(beforeAndAfters)); }
public BenchmarkRunEstimate GetRunEstimate <TBenchmarkContainer>( IEnumerable <ISampleSizeDeterminer> sampleSizeDeterminers) { // TODO: Point of this class was to cache - needs to cache :) // TODO: P2 - Get the run time for the preliminary FROM a simple invocation that sees how long the methods take. // E.G. Do not limit to 10s if the method itself takes 30s var preliminaryRunParameters = new BenchmarkRunParameters(desiredMaxLatency: TimeSpan.FromSeconds(10)); var preliminaryRunResults = this.RunBenchmark <TBenchmarkContainer>(preliminaryRunParameters); IReadOnlyDictionary <ISampleSizeDeterminer, TimeSpan> estimatedTimeBySource = sampleSizeDeterminers .ToDictionary( sampleSizeDeterminer => sampleSizeDeterminer, sampleSizeDeterminer => { // The time taken for the validator is defined as // Sum( // each: parameterCase // case -> // iterationCount = max(treatmentIterationCountRequired, baselineIterationCountRequired) // iterationCount * treatmentTime + iterationCount * baselineTime // ) TimeSpan estimatedDuration = TimeSpan.Zero; foreach (var resultAndCase in preliminaryRunResults.ResultsByCase) { var benchmarkCase = resultAndCase.Key; var baselineReport = resultAndCase.Value.Baseline; var treatmentReport = resultAndCase.Value.Treatment; var sampleSizeRequirementForBaseline = sampleSizeDeterminer.GetSampleSizeRequirement(resultAndCase.Value).SamplesForBaseline; var sampleSizeRequirementForTreatment = sampleSizeDeterminer.GetSampleSizeRequirement(resultAndCase.Value).SamplesForTreatment; // As of now, we run both for the same amount of samples sampleSizeRequirementForBaseline = Math.Max((int)sampleSizeRequirementForBaseline, (int)sampleSizeRequirementForTreatment); sampleSizeRequirementForTreatment = Math.Max((int)sampleSizeRequirementForBaseline, (int)sampleSizeRequirementForTreatment); var nanosecondsPerSampleInBaseline = baselineReport .GetResultRuns() .Average(workloadMeasurement => workloadMeasurement.Nanoseconds); var timePerSampleInBaseline = TimeSpan.FromTicks((long)(nanosecondsPerSampleInBaseline / 100)); var nanosecondsPerSampleInTreatment = treatmentReport .GetResultRuns() .Average(workloadMeasurement => workloadMeasurement.Nanoseconds); var timePerSampleInTreatment = TimeSpan.FromTicks((long)(nanosecondsPerSampleInTreatment / 100)); var totalTimeForBaseline = TimeSpan.FromMilliseconds(timePerSampleInBaseline.TotalMilliseconds * sampleSizeRequirementForBaseline); var totalTimeForTreatment = TimeSpan.FromMilliseconds(timePerSampleInTreatment.TotalMilliseconds * sampleSizeRequirementForTreatment); var totalTimeForCase = totalTimeForBaseline + totalTimeForTreatment; estimatedDuration += totalTimeForCase; } return(estimatedDuration); }); var maxTime = estimatedTimeBySource.Values.Max(); // TODO: P2 - Need to encapsulate the information about each parameters and how many ITERATIONS they should run for from above BenchmarkRunParameters runParameters = new BenchmarkRunParameters(maxTime); return(new BenchmarkRunEstimate( maxTime, runParameters, estimatedTimeBySource)); }
public BenchmarkRunEstimate(TimeSpan estimatedTime, BenchmarkRunParameters runParameters, IReadOnlyDictionary <ISampleSizeDeterminer, TimeSpan> estimatedTimeBySource) { this.EstimatedTime = estimatedTime; this.RunParameters = runParameters; this.EstimatedTimeBySource = estimatedTimeBySource; }