public static void ValidateSummary( Summary summary, double defaultMinRatio, double defaultMaxRatio, CompetitionTargets competitionTargets) { // Based on 95th percentile const double percentileRatio = 0.95; var benchmarkGroups = summary.SameConditionBenchmarks(); foreach (var benchmarkGroup in benchmarkGroups) { var baselineBenchmark = GetBaselineBenchmark(benchmarkGroup); var baselineMetric = summary.GetPercentile(baselineBenchmark, percentileRatio); if (baselineMetric <= 0) { throw new InvalidOperationException($"Baseline benchmark {baselineBenchmark.ShortInfo} does not compute"); } foreach (var benchmark in benchmarkGroup) { if (benchmark == baselineBenchmark) { continue; } CompetitionTarget competitionTarget; if (!competitionTargets.TryGetValue(benchmark.Target, out competitionTarget)) { continue; } var benchmarkMinRatio = defaultMinRatio; if (!competitionTarget.MinIsEmpty) { benchmarkMinRatio = competitionTarget.Min; } var benchmarkMaxRatio = defaultMaxRatio; if (!competitionTarget.MaxIsEmpty) { benchmarkMaxRatio = competitionTarget.Max; } var reportMetric = summary.GetPercentile(benchmark, percentileRatio); var actualRatio = Math.Round(reportMetric / baselineMetric, 2); ValidateBenchmark(benchmark, actualRatio, benchmarkMinRatio, benchmarkMaxRatio); } } }
public static void InitCompetitionTargets(CompetitionTargets competitionTargets, Summary summary) { competitionTargets.Clear(); var resourceCache = new Dictionary <string, XDocument>(); foreach (var target in summary.Benchmarks.Select(d => d.Target).Distinct()) { var competitionAttribute = (CompetitionBenchmarkAttribute) target.Method.GetCustomAttribute(typeof(CompetitionBenchmarkAttribute)); if (competitionAttribute != null && !competitionAttribute.Baseline && !competitionAttribute.DoesNotCompete) { var competitionTarget = GetCompetitionTarget(target, competitionAttribute, resourceCache); competitionTargets.Add(target, competitionTarget); } } }
public static CompetitionTarget[] GetNewCompetitionTargets(Summary summary, CompetitionTargets competitionTargets) { var fixedMinTargets = new HashSet <CompetitionTarget>(); var fixedMaxTargets = new HashSet <CompetitionTarget>(); var newTargets = new Dictionary <Target, CompetitionTarget>(); foreach (var benchGroup in summary.SameConditionBenchmarks()) { var baselineBenchmark = GetBaselineBenchmark(benchGroup); var baselineMetricMin = summary.GetPercentile(baselineBenchmark, 0.85); var baselineMetricMax = summary.GetPercentile(baselineBenchmark, 0.95); foreach (var benchmark in benchGroup) { if (benchmark == baselineBenchmark) { continue; } CompetitionTarget competitionTarget; if (!competitionTargets.TryGetValue(benchmark.Target, out competitionTarget)) { continue; } var minRatio = summary.GetPercentile(benchmark, 0.85) / baselineMetricMin; var maxRatio = summary.GetPercentile(benchmark, 0.95) / baselineMetricMax; if (minRatio > maxRatio) { Algorithms.Swap(ref minRatio, ref maxRatio); } CompetitionTarget newTarget; if (!newTargets.TryGetValue(competitionTarget.Target, out newTarget)) { newTarget = competitionTarget.Clone(); } if (newTarget.UnionWithMin(minRatio)) { fixedMinTargets.Add(newTarget); newTargets[newTarget.Target] = newTarget; } if (newTarget.UnionWithMax(maxRatio)) { fixedMaxTargets.Add(newTarget); newTargets[newTarget.Target] = newTarget; } } } foreach (var competitionTarget in fixedMinTargets) { // min = 0.97x; competitionTarget.UnionWithMin( Math.Floor(competitionTarget.Min * 97) / 100); } foreach (var competitionTarget in fixedMaxTargets) { // max = 1.03x; competitionTarget.UnionWithMax( Math.Ceiling(competitionTarget.Max * 103) / 100); } return(newTargets.Values.ToArray()); }