public static void ValidateSummary(
Summary summary, double defaultMinRatio, double defaultMaxRatio,
CompetitionTargets competitionTargets)
{
// Based on 95th percentile
const double percentileRatio = 0.95;
var benchmarkGroups = summary.SameConditionBenchmarks();
foreach (var benchmarkGroup in benchmarkGroups)
{
var baselineBenchmark = GetBaselineBenchmark(benchmarkGroup);
var baselineMetric = summary.GetPercentile(baselineBenchmark, percentileRatio);
if (baselineMetric <= 0)
{
throw new InvalidOperationException($"Baseline benchmark {baselineBenchmark.ShortInfo} does not compute");
}
foreach (var benchmark in benchmarkGroup)
{
if (benchmark == baselineBenchmark)
{
continue;
}
CompetitionTarget competitionTarget;
if (!competitionTargets.TryGetValue(benchmark.Target, out competitionTarget))
{
continue;
}
var benchmarkMinRatio = defaultMinRatio;
if (!competitionTarget.MinIsEmpty)
{
benchmarkMinRatio = competitionTarget.Min;
}
var benchmarkMaxRatio = defaultMaxRatio;
if (!competitionTarget.MaxIsEmpty)
{
benchmarkMaxRatio = competitionTarget.Max;
}
var reportMetric = summary.GetPercentile(benchmark, percentileRatio);
var actualRatio = Math.Round(reportMetric / baselineMetric, 2);
ValidateBenchmark(benchmark, actualRatio, benchmarkMinRatio, benchmarkMaxRatio);
}
}
}
public static void InitCompetitionTargets(CompetitionTargets competitionTargets, Summary summary)
{
competitionTargets.Clear();
var resourceCache = new Dictionary <string, XDocument>();
foreach (var target in summary.Benchmarks.Select(d => d.Target).Distinct())
{
var competitionAttribute = (CompetitionBenchmarkAttribute)
target.Method.GetCustomAttribute(typeof(CompetitionBenchmarkAttribute));
if (competitionAttribute != null &&
!competitionAttribute.Baseline &&
!competitionAttribute.DoesNotCompete)
{
var competitionTarget = GetCompetitionTarget(target, competitionAttribute, resourceCache);
competitionTargets.Add(target, competitionTarget);
}
}
}
public static CompetitionTarget[] GetNewCompetitionTargets(Summary summary, CompetitionTargets competitionTargets)
{
var fixedMinTargets = new HashSet <CompetitionTarget>();
var fixedMaxTargets = new HashSet <CompetitionTarget>();
var newTargets = new Dictionary <Target, CompetitionTarget>();
foreach (var benchGroup in summary.SameConditionBenchmarks())
{
var baselineBenchmark = GetBaselineBenchmark(benchGroup);
var baselineMetricMin = summary.GetPercentile(baselineBenchmark, 0.85);
var baselineMetricMax = summary.GetPercentile(baselineBenchmark, 0.95);
foreach (var benchmark in benchGroup)
{
if (benchmark == baselineBenchmark)
{
continue;
}
CompetitionTarget competitionTarget;
if (!competitionTargets.TryGetValue(benchmark.Target, out competitionTarget))
{
continue;
}
var minRatio = summary.GetPercentile(benchmark, 0.85) / baselineMetricMin;
var maxRatio = summary.GetPercentile(benchmark, 0.95) / baselineMetricMax;
if (minRatio > maxRatio)
{
Algorithms.Swap(ref minRatio, ref maxRatio);
}
CompetitionTarget newTarget;
if (!newTargets.TryGetValue(competitionTarget.Target, out newTarget))
{
newTarget = competitionTarget.Clone();
}
if (newTarget.UnionWithMin(minRatio))
{
fixedMinTargets.Add(newTarget);
newTargets[newTarget.Target] = newTarget;
}
if (newTarget.UnionWithMax(maxRatio))
{
fixedMaxTargets.Add(newTarget);
newTargets[newTarget.Target] = newTarget;
}
}
}
foreach (var competitionTarget in fixedMinTargets)
{
// min = 0.97x;
competitionTarget.UnionWithMin(
Math.Floor(competitionTarget.Min * 97) / 100);
}
foreach (var competitionTarget in fixedMaxTargets)
{
// max = 1.03x;
competitionTarget.UnionWithMax(
Math.Ceiling(competitionTarget.Max * 103) / 100);
}
return(newTargets.Values.ToArray());
}
