private double TestStatistic(ExperimentPerformance classifier1, ExperimentPerformance classifier2)
{
var difference = new double[classifier1.NumberOfExperiments()];
for (var i = 0; i < classifier1.NumberOfExperiments(); i++)
{
difference[i] = classifier1.GetErrorRate(i) - classifier2.GetErrorRate(i);
}
double denominator = 0;
double numerator = 0;
for (var i = 0; i < classifier1.NumberOfExperiments() / 2; i++)
{
var mean = (difference[2 * i] + difference[2 * i + 1]) / 2;
numerator += mean;
var variance = (difference[2 * i] - mean) * (difference[2 * i] - mean) +
(difference[2 * i + 1] - mean) * (difference[2 * i + 1] - mean);
denominator += variance;
}
numerator = System.Math.Sqrt(10) * numerator / 5;
denominator = System.Math.Sqrt(denominator / 5);
return(numerator / denominator);
}
public override StatisticalTestResult Compare(ExperimentPerformance classifier1,
ExperimentPerformance classifier2)
{
var statistic = TestStatistic(classifier1, classifier2);
var degreeOfFreedom1 = classifier1.NumberOfExperiments();
var degreeOfFreedom2 = classifier1.NumberOfExperiments() / 2;
return(new StatisticalTestResult(Distribution.FDistribution(statistic, degreeOfFreedom1, degreeOfFreedom2),
true
));
}
public override StatisticalTestResult Compare(ExperimentPerformance classifier1,
ExperimentPerformance classifier2)
{
int plus = 0, minus = 0;
for (var i = 0; i < classifier1.NumberOfExperiments(); i++)
{
if (classifier1.GetErrorRate(i) < classifier2.GetErrorRate(i))
{
plus++;
}
else
{
if (classifier1.GetErrorRate(i) > classifier2.GetErrorRate(i))
{
minus++;
}
}
}
var total = plus + minus;
var pValue = 0.0;
for (var i = 0; i <= plus; i++)
{
pValue += Binomial(total, i) / System.Math.Pow(2, total);
}
return(new StatisticalTestResult(pValue, false));
}
private double TestStatistic(ExperimentPerformance classifier1, ExperimentPerformance classifier2)
{
var difference = new double[classifier1.NumberOfExperiments()];
var sum = 0.0;
for (var i = 0; i < classifier1.NumberOfExperiments(); i++)
{
difference[i] = classifier1.GetErrorRate(i) - classifier2.GetErrorRate(i);
sum += difference[i];
}
var mean = sum / classifier1.NumberOfExperiments();
sum = 0.0;
for (var i = 0; i < classifier1.NumberOfExperiments(); i++)
{
sum += (difference[i] - mean) * (difference[i] - mean);
}
var standardDeviation = System.Math.Sqrt(sum / (classifier1.NumberOfExperiments() - 1));
return(System.Math.Sqrt(classifier1.NumberOfExperiments()) * mean / standardDeviation);
}
