private static List<TestResultComparison> DoComparisons(List<Tuple<string, string>> allComparisonIds, Dictionary<string, Dictionary<string, TestResult>> testResults)
{
var comparisonResults = new List<TestResultComparison>();
foreach (var comparisonIds in allComparisonIds)
{
var baseline = testResults[comparisonIds.Item1];
var comparison = testResults[comparisonIds.Item2];
foreach (var comparisonTest in comparison.Values)
{
var baselineTest = baseline[comparisonTest.TestName];
var comparisonResult = new TestResultComparison();
comparisonResult.BaselineResult = baselineTest;
comparisonResult.ComparisonResult = comparisonTest;
comparisonResult.TestName = comparisonTest.TestName;
var MetricComparisons = new Dictionary<string, MetricComparison>();
foreach (var metric in comparisonTest.Iterations.First().MetricValues.Keys)
{
try
{
var metricComparison = new MetricComparison(comparisonResult, metric);
// Compute the standard error in the difference
var baselineCount = baselineTest.Iterations.Count;
var baselineSum = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metric]);
var baselineSumSquared = baselineSum * baselineSum;
var baselineSumOfSquares = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metric] * iteration.MetricValues[metric]);
var comparisonCount = comparisonTest.Iterations.Count;
var comparisonSum = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metric]);
var comparisonSumSquared = comparisonSum * comparisonSum;
var comparisonSumOfSquares = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metric] * iteration.MetricValues[metric]);
var stdErrorDiff = Math.Sqrt((baselineSumOfSquares - (baselineSumSquared / baselineCount) + comparisonSumOfSquares - (comparisonSumSquared / comparisonCount)) * (1.0 / baselineCount + 1.0 / comparisonCount) / (baselineCount + comparisonCount - 1));
var interval = stdErrorDiff * MathNet.Numerics.ExcelFunctions.TInv(1.0 - ErrorConfidence, baselineCount + comparisonCount - 2);
metricComparison.PercentChange = (comparisonTest.Stats[metric].Mean - baselineTest.Stats[metric].Mean) / baselineTest.Stats[metric].Mean;
metricComparison.PercentChangeError = interval / baselineTest.Stats[metric].Mean;
MetricComparisons.Add(metric, metricComparison);
}
catch
{ // metric does not exist in either baseline or comparison. Ignore comparison.
}
}
comparisonResult.MetricComparisons = MetricComparisons;
comparisonResults.Add(comparisonResult);
}
}
return comparisonResults;
}
private static List <TestResultComparison> DoComparisons(List <Tuple <string, string> > allComparisonIds, Dictionary <string, Dictionary <string, TestResult> > testResults)
{
var comparisonResults = new List <TestResultComparison>();
foreach (var comparisonIds in allComparisonIds)
{
var baseline = testResults[comparisonIds.Item1];
var comparison = testResults[comparisonIds.Item2];
foreach (var comparisonTest in comparison.Values)
{
var baselineTest = baseline[comparisonTest.TestName];
// Compute the standard error in the difference
var baselineCount = baselineTest.Iterations.Count;
var baselineSum = baselineTest.Iterations.Sum(iteration => iteration.MetricValues["Duration"]);
var baselineSumSquared = baselineSum * baselineSum;
var baselineSumOfSquares = baselineTest.Iterations.Sum(iteration => iteration.MetricValues["Duration"] * iteration.MetricValues["Duration"]);
var comparisonCount = comparisonTest.Iterations.Count;
var comparisonSum = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues["Duration"]);
var comparisonSumSquared = comparisonSum * comparisonSum;
var comparisonSumOfSquares = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues["Duration"] * iteration.MetricValues["Duration"]);
var stdErrorDiff = Math.Sqrt((baselineSumOfSquares - (baselineSumSquared / baselineCount) + comparisonSumOfSquares - (comparisonSumSquared / comparisonCount)) * (1.0 / baselineCount + 1.0 / comparisonCount) / (baselineCount + comparisonCount - 1));
var interval = stdErrorDiff * MathNet.Numerics.ExcelFunctions.TInv(1.0 - ErrorConfidence, baselineCount + comparisonCount - 2);
var comparisonResult = new TestResultComparison();
comparisonResult.BaselineResult = baselineTest;
comparisonResult.ComparisonResult = comparisonTest;
comparisonResult.TestName = comparisonTest.TestName;
comparisonResult.PercentChange = (comparisonTest.Stats["Duration"].Mean - baselineTest.Stats["Duration"].Mean) / baselineTest.Stats["Duration"].Mean;
comparisonResult.PercentChangeError = interval / baselineTest.Stats["Duration"].Mean;
comparisonResults.Add(comparisonResult);
}
}
return(comparisonResults);
}
private const double ErrorConfidence = 0.95; // TODO: make configurable
#endregion Fields
#region Methods
private static List<TestResultComparison> DoComparisons(List<Tuple<string, string>> allComparisonIds, Dictionary<string, Dictionary<string, TestResult>> testResults)
{
var comparisonResults = new List<TestResultComparison>();
foreach (var comparisonIds in allComparisonIds)
{
var baseline = testResults[comparisonIds.Item1];
var comparison = testResults[comparisonIds.Item2];
foreach (var comparisonTest in comparison.Values)
{
var baselineTest = baseline[comparisonTest.TestName];
// Compute the standard error in the difference
var baselineCount = baselineTest.Iterations.Count;
var baselineSum = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[DurationMetricName]);
var baselineSumSquared = baselineSum * baselineSum;
var baselineSumOfSquares = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[DurationMetricName] * iteration.MetricValues[DurationMetricName]);
var comparisonCount = comparisonTest.Iterations.Count;
var comparisonSum = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[DurationMetricName]);
var comparisonSumSquared = comparisonSum * comparisonSum;
var comparisonSumOfSquares = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[DurationMetricName] * iteration.MetricValues[DurationMetricName]);
var stdErrorDiff = Math.Sqrt((baselineSumOfSquares - (baselineSumSquared / baselineCount) + comparisonSumOfSquares - (comparisonSumSquared / comparisonCount)) * (1.0 / baselineCount + 1.0 / comparisonCount) / (baselineCount + comparisonCount - 1));
var interval = stdErrorDiff * MathNet.Numerics.ExcelFunctions.TInv(1.0 - ErrorConfidence, baselineCount + comparisonCount - 2);
var comparisonResult = new TestResultComparison();
comparisonResult.BaselineResult = baselineTest;
comparisonResult.ComparisonResult = comparisonTest;
comparisonResult.TestName = comparisonTest.TestName;
comparisonResult.PercentChange = (comparisonTest.Stats[DurationMetricName].Mean - baselineTest.Stats[DurationMetricName].Mean) / baselineTest.Stats[DurationMetricName].Mean;
comparisonResult.PercentChangeError = interval / baselineTest.Stats[DurationMetricName].Mean;
comparisonResults.Add(comparisonResult);
}
}
return comparisonResults;
}
private static Dictionary <string, List <TestResultComparison> > DoComparisons(List <Tuple <string, string> > allComparisonIds, Dictionary <string, Dictionary <string, TestResult> > testResults)
{
var comparisonMatrix = new Dictionary <string, List <TestResultComparison> >();
foreach (var comparisonIds in allComparisonIds)
{
var baseline = testResults[comparisonIds.Item1];
var comparison = testResults[comparisonIds.Item2];
//
// Analize results metric-by-metric
//
foreach (var metricName in Properties.AllMetrics.Keys)
{
var comparisonResults = new List <TestResultComparison>();
comparisonMatrix.Add(metricName, comparisonResults);
foreach (var comparisonTest in comparison.Values)
{
var baselineTest = baseline[comparisonTest.TestName];
var baselineCount = baselineTest.Iterations.Count;
var comparisonCount = comparisonTest.Iterations.Count;
if (baselineCount <= 0 || comparisonCount <= 0)
{
continue;
}
if (!baselineTest.Iterations[0].MetricValues.ContainsKey(metricName) ||
!comparisonTest.Iterations[0].MetricValues.ContainsKey(metricName))
{
continue;
}
// Compute the standard error in the difference
var baselineSum = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metricName]);
var baselineSumSquared = baselineSum * baselineSum;
var baselineSumOfSquares = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metricName] * iteration.MetricValues[metricName]);
var comparisonSum = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metricName]);
var comparisonSumSquared = comparisonSum * comparisonSum;
var comparisonSumOfSquares = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metricName] * iteration.MetricValues[metricName]);
var stdErrorDiff = Math.Sqrt((baselineSumOfSquares - (baselineSumSquared / baselineCount) + comparisonSumOfSquares - (comparisonSumSquared / comparisonCount)) * (1.0 / baselineCount + 1.0 / comparisonCount) / (baselineCount + comparisonCount - 1));
var interval = stdErrorDiff * MathNet.Numerics.ExcelFunctions.TInv(1.0 - Properties.ErrorConfidence, baselineCount + comparisonCount - 2);
RunningStatistics comparisonStats = comparisonTest.Stats[metricName].RunningStatistics;
RunningStatistics baselineStats = baselineTest.Stats[metricName].RunningStatistics;
var comparisonResult = new TestResultComparison();
comparisonResult.BaselineResult = baselineTest;
comparisonResult.ComparisonResult = comparisonTest;
comparisonResult.TestName = comparisonTest.TestName;
comparisonResult.PercentChange = (comparisonStats.Mean - baselineStats.Mean) / baselineStats.Mean;
comparisonResult.PercentChangeError = interval / baselineStats.Mean;
comparisonResults.Add(comparisonResult);
}
}
}
return(comparisonMatrix);
}
private static Dictionary<string, List<TestResultComparison>> DoComparisons(List<Tuple<string, string>> allComparisonIds, Dictionary<string, Dictionary<string, TestResult>> testResults)
{
var comparisonMatrix = new Dictionary<string, List<TestResultComparison>>();
foreach (var comparisonIds in allComparisonIds)
{
var baseline = testResults[comparisonIds.Item1];
var comparison = testResults[comparisonIds.Item2];
//
// Analize results metric-by-metric
//
foreach (var metricName in Properties.AllMetrics.Keys)
{
var comparisonResults = new List<TestResultComparison>();
comparisonMatrix.Add(metricName, comparisonResults);
foreach (var comparisonTest in comparison.Values)
{
var baselineTest = baseline[comparisonTest.TestName];
var baselineCount = baselineTest.Iterations.Count;
var comparisonCount = comparisonTest.Iterations.Count;
if (baselineCount <= 0 || comparisonCount <= 0)
{
continue;
}
if (!baselineTest.Iterations[0].MetricValues.ContainsKey(metricName) ||
!comparisonTest.Iterations[0].MetricValues.ContainsKey(metricName))
{
continue;
}
// Compute the standard error in the difference
var baselineSum = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metricName]);
var baselineSumSquared = baselineSum * baselineSum;
var baselineSumOfSquares = baselineTest.Iterations.Sum(iteration => iteration.MetricValues[metricName] * iteration.MetricValues[metricName]);
var comparisonSum = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metricName]);
var comparisonSumSquared = comparisonSum * comparisonSum;
var comparisonSumOfSquares = comparisonTest.Iterations.Sum(iteration => iteration.MetricValues[metricName] * iteration.MetricValues[metricName]);
var stdErrorDiff = Math.Sqrt((baselineSumOfSquares - (baselineSumSquared / baselineCount) + comparisonSumOfSquares - (comparisonSumSquared / comparisonCount)) * (1.0 / baselineCount + 1.0 / comparisonCount) / (baselineCount + comparisonCount - 1));
var interval = stdErrorDiff * MathNet.Numerics.ExcelFunctions.TInv(1.0 - Properties.ErrorConfidence, baselineCount + comparisonCount - 2);
RunningStatistics comparisonStats = comparisonTest.Stats[metricName].RunningStatistics;
RunningStatistics baselineStats = baselineTest.Stats[metricName].RunningStatistics;
var comparisonResult = new TestResultComparison();
comparisonResult.BaselineResult = baselineTest;
comparisonResult.ComparisonResult = comparisonTest;
comparisonResult.TestName = comparisonTest.TestName;
comparisonResult.PercentChange = (comparisonStats.Mean - baselineStats.Mean) / baselineStats.Mean;
comparisonResult.PercentChangeError = interval / baselineStats.Mean;
comparisonResults.Add(comparisonResult);
}
}
}
return comparisonMatrix;
}
public MetricComparison(TestResultComparison parent, string metricName) { this.parent = parent; this.MetricName = metricName; }
