private static List<TestResultComparison> DoComparisons(List<Tuple<string, string>> allComparisonIds, Dictionary<string, Dictionary<string, TestResult>> testResults) { var comparisonResults = new List<TestResultComparison>(); foreach (var comparisonIds in allComparisonIds) { var baseline = testResults[comparisonIds.Item1]; var comparison = testResults[comparisonIds.Item2]; foreach (var comparisonTest in comparison.Values) { var baselineTest = baseline[comparisonTest.TestName]; var comparisonResult = new TestResultComparison(); comparisonResult.BaselineResult = baselineTest; comparisonResult.ComparisonResult = comparisonTest; comparisonResult.TestName = comparisonTest.TestName; var MetricComparisons = new Dictionary<string, MetricComparison>(); foreach (var metric in comparisonTest.Iterations.First().MetricValues.Keys) { try { var metricComparison = new MetricComparison(comparisonResult, metric); // Compute the standard error in the difference var baselineCount = baselineTest.Iterations.Count; var baselineSum = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metric]); var baselineSumSquared = baselineSum * baselineSum; var baselineSumOfSquares = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metric] * iteration.MetricValues[metric]); var comparisonCount = comparisonTest.Iterations.Count; var comparisonSum = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metric]); var comparisonSumSquared = comparisonSum * comparisonSum; var comparisonSumOfSquares = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metric] * iteration.MetricValues[metric]); var stdErrorDiff = Math.Sqrt((baselineSumOfSquares - (baselineSumSquared / baselineCount) + comparisonSumOfSquares - (comparisonSumSquared / comparisonCount)) * (1.0 / baselineCount + 1.0 / comparisonCount) / (baselineCount + comparisonCount - 1)); var interval = stdErrorDiff * MathNet.Numerics.ExcelFunctions.TInv(1.0 - ErrorConfidence, baselineCount + comparisonCount - 2); metricComparison.PercentChange = (comparisonTest.Stats[metric].Mean - baselineTest.Stats[metric].Mean) / baselineTest.Stats[metric].Mean; metricComparison.PercentChangeError = interval / baselineTest.Stats[metric].Mean; MetricComparisons.Add(metric, metricComparison); } catch { // metric does not exist in either baseline or comparison. Ignore comparison. } } comparisonResult.MetricComparisons = MetricComparisons; comparisonResults.Add(comparisonResult); } } return comparisonResults; }
private static List <TestResultComparison> DoComparisons(List <Tuple <string, string> > allComparisonIds, Dictionary <string, Dictionary <string, TestResult> > testResults) { var comparisonResults = new List <TestResultComparison>(); foreach (var comparisonIds in allComparisonIds) { var baseline = testResults[comparisonIds.Item1]; var comparison = testResults[comparisonIds.Item2]; foreach (var comparisonTest in comparison.Values) { var baselineTest = baseline[comparisonTest.TestName]; // Compute the standard error in the difference var baselineCount = baselineTest.Iterations.Count; var baselineSum = baselineTest.Iterations.Sum(iteration => iteration.MetricValues["Duration"]); var baselineSumSquared = baselineSum * baselineSum; var baselineSumOfSquares = baselineTest.Iterations.Sum(iteration => iteration.MetricValues["Duration"] * iteration.MetricValues["Duration"]); var comparisonCount = comparisonTest.Iterations.Count; var comparisonSum = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues["Duration"]); var comparisonSumSquared = comparisonSum * comparisonSum; var comparisonSumOfSquares = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues["Duration"] * iteration.MetricValues["Duration"]); var stdErrorDiff = Math.Sqrt((baselineSumOfSquares - (baselineSumSquared / baselineCount) + comparisonSumOfSquares - (comparisonSumSquared / comparisonCount)) * (1.0 / baselineCount + 1.0 / comparisonCount) / (baselineCount + comparisonCount - 1)); var interval = stdErrorDiff * MathNet.Numerics.ExcelFunctions.TInv(1.0 - ErrorConfidence, baselineCount + comparisonCount - 2); var comparisonResult = new TestResultComparison(); comparisonResult.BaselineResult = baselineTest; comparisonResult.ComparisonResult = comparisonTest; comparisonResult.TestName = comparisonTest.TestName; comparisonResult.PercentChange = (comparisonTest.Stats["Duration"].Mean - baselineTest.Stats["Duration"].Mean) / baselineTest.Stats["Duration"].Mean; comparisonResult.PercentChangeError = interval / baselineTest.Stats["Duration"].Mean; comparisonResults.Add(comparisonResult); } } return(comparisonResults); }
private const double ErrorConfidence = 0.95; // TODO: make configurable #endregion Fields #region Methods private static List<TestResultComparison> DoComparisons(List<Tuple<string, string>> allComparisonIds, Dictionary<string, Dictionary<string, TestResult>> testResults) { var comparisonResults = new List<TestResultComparison>(); foreach (var comparisonIds in allComparisonIds) { var baseline = testResults[comparisonIds.Item1]; var comparison = testResults[comparisonIds.Item2]; foreach (var comparisonTest in comparison.Values) { var baselineTest = baseline[comparisonTest.TestName]; // Compute the standard error in the difference var baselineCount = baselineTest.Iterations.Count; var baselineSum = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[DurationMetricName]); var baselineSumSquared = baselineSum * baselineSum; var baselineSumOfSquares = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[DurationMetricName] * iteration.MetricValues[DurationMetricName]); var comparisonCount = comparisonTest.Iterations.Count; var comparisonSum = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[DurationMetricName]); var comparisonSumSquared = comparisonSum * comparisonSum; var comparisonSumOfSquares = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[DurationMetricName] * iteration.MetricValues[DurationMetricName]); var stdErrorDiff = Math.Sqrt((baselineSumOfSquares - (baselineSumSquared / baselineCount) + comparisonSumOfSquares - (comparisonSumSquared / comparisonCount)) * (1.0 / baselineCount + 1.0 / comparisonCount) / (baselineCount + comparisonCount - 1)); var interval = stdErrorDiff * MathNet.Numerics.ExcelFunctions.TInv(1.0 - ErrorConfidence, baselineCount + comparisonCount - 2); var comparisonResult = new TestResultComparison(); comparisonResult.BaselineResult = baselineTest; comparisonResult.ComparisonResult = comparisonTest; comparisonResult.TestName = comparisonTest.TestName; comparisonResult.PercentChange = (comparisonTest.Stats[DurationMetricName].Mean - baselineTest.Stats[DurationMetricName].Mean) / baselineTest.Stats[DurationMetricName].Mean; comparisonResult.PercentChangeError = interval / baselineTest.Stats[DurationMetricName].Mean; comparisonResults.Add(comparisonResult); } } return comparisonResults; }
private static Dictionary <string, List <TestResultComparison> > DoComparisons(List <Tuple <string, string> > allComparisonIds, Dictionary <string, Dictionary <string, TestResult> > testResults) { var comparisonMatrix = new Dictionary <string, List <TestResultComparison> >(); foreach (var comparisonIds in allComparisonIds) { var baseline = testResults[comparisonIds.Item1]; var comparison = testResults[comparisonIds.Item2]; // // Analize results metric-by-metric // foreach (var metricName in Properties.AllMetrics.Keys) { var comparisonResults = new List <TestResultComparison>(); comparisonMatrix.Add(metricName, comparisonResults); foreach (var comparisonTest in comparison.Values) { var baselineTest = baseline[comparisonTest.TestName]; var baselineCount = baselineTest.Iterations.Count; var comparisonCount = comparisonTest.Iterations.Count; if (baselineCount <= 0 || comparisonCount <= 0) { continue; } if (!baselineTest.Iterations[0].MetricValues.ContainsKey(metricName) || !comparisonTest.Iterations[0].MetricValues.ContainsKey(metricName)) { continue; } // Compute the standard error in the difference var baselineSum = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metricName]); var baselineSumSquared = baselineSum * baselineSum; var baselineSumOfSquares = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metricName] * iteration.MetricValues[metricName]); var comparisonSum = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metricName]); var comparisonSumSquared = comparisonSum * comparisonSum; var comparisonSumOfSquares = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metricName] * iteration.MetricValues[metricName]); var stdErrorDiff = Math.Sqrt((baselineSumOfSquares - (baselineSumSquared / baselineCount) + comparisonSumOfSquares - (comparisonSumSquared / comparisonCount)) * (1.0 / baselineCount + 1.0 / comparisonCount) / (baselineCount + comparisonCount - 1)); var interval = stdErrorDiff * MathNet.Numerics.ExcelFunctions.TInv(1.0 - Properties.ErrorConfidence, baselineCount + comparisonCount - 2); RunningStatistics comparisonStats = comparisonTest.Stats[metricName].RunningStatistics; RunningStatistics baselineStats = baselineTest.Stats[metricName].RunningStatistics; var comparisonResult = new TestResultComparison(); comparisonResult.BaselineResult = baselineTest; comparisonResult.ComparisonResult = comparisonTest; comparisonResult.TestName = comparisonTest.TestName; comparisonResult.PercentChange = (comparisonStats.Mean - baselineStats.Mean) / baselineStats.Mean; comparisonResult.PercentChangeError = interval / baselineStats.Mean; comparisonResults.Add(comparisonResult); } } } return(comparisonMatrix); }
private static Dictionary<string, List<TestResultComparison>> DoComparisons(List<Tuple<string, string>> allComparisonIds, Dictionary<string, Dictionary<string, TestResult>> testResults) { var comparisonMatrix = new Dictionary<string, List<TestResultComparison>>(); foreach (var comparisonIds in allComparisonIds) { var baseline = testResults[comparisonIds.Item1]; var comparison = testResults[comparisonIds.Item2]; // // Analize results metric-by-metric // foreach (var metricName in Properties.AllMetrics.Keys) { var comparisonResults = new List<TestResultComparison>(); comparisonMatrix.Add(metricName, comparisonResults); foreach (var comparisonTest in comparison.Values) { var baselineTest = baseline[comparisonTest.TestName]; var baselineCount = baselineTest.Iterations.Count; var comparisonCount = comparisonTest.Iterations.Count; if (baselineCount <= 0 || comparisonCount <= 0) { continue; } if (!baselineTest.Iterations[0].MetricValues.ContainsKey(metricName) || !comparisonTest.Iterations[0].MetricValues.ContainsKey(metricName)) { continue; } // Compute the standard error in the difference var baselineSum = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metricName]); var baselineSumSquared = baselineSum * baselineSum; var baselineSumOfSquares = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metricName] * iteration.MetricValues[metricName]); var comparisonSum = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metricName]); var comparisonSumSquared = comparisonSum * comparisonSum; var comparisonSumOfSquares = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metricName] * iteration.MetricValues[metricName]); var stdErrorDiff = Math.Sqrt((baselineSumOfSquares - (baselineSumSquared / baselineCount) + comparisonSumOfSquares - (comparisonSumSquared / comparisonCount)) * (1.0 / baselineCount + 1.0 / comparisonCount) / (baselineCount + comparisonCount - 1)); var interval = stdErrorDiff * MathNet.Numerics.ExcelFunctions.TInv(1.0 - Properties.ErrorConfidence, baselineCount + comparisonCount - 2); RunningStatistics comparisonStats = comparisonTest.Stats[metricName].RunningStatistics; RunningStatistics baselineStats = baselineTest.Stats[metricName].RunningStatistics; var comparisonResult = new TestResultComparison(); comparisonResult.BaselineResult = baselineTest; comparisonResult.ComparisonResult = comparisonTest; comparisonResult.TestName = comparisonTest.TestName; comparisonResult.PercentChange = (comparisonStats.Mean - baselineStats.Mean) / baselineStats.Mean; comparisonResult.PercentChangeError = interval / baselineStats.Mean; comparisonResults.Add(comparisonResult); } } } return comparisonMatrix; }
public MetricComparison(TestResultComparison parent, string metricName) { this.parent = parent; this.MetricName = metricName; }