public override StatisticalTestResult Compare(ExperimentPerformance classifier1,
ExperimentPerformance classifier2)
{
int plus = 0, minus = 0;
for (var i = 0; i < classifier1.NumberOfExperiments(); i++)
{
if (classifier1.GetErrorRate(i) < classifier2.GetErrorRate(i))
{
plus++;
}
else
{
if (classifier1.GetErrorRate(i) > classifier2.GetErrorRate(i))
{
minus++;
}
}
}
var total = plus + minus;
var pValue = 0.0;
for (var i = 0; i <= plus; i++)
{
pValue += Binomial(total, i) / System.Math.Pow(2, total);
}
return(new StatisticalTestResult(pValue, false));
}
private double TestStatistic(ExperimentPerformance classifier1, ExperimentPerformance classifier2)
{
var difference = new double[classifier1.NumberOfExperiments()];
for (var i = 0; i < classifier1.NumberOfExperiments(); i++)
{
difference[i] = classifier1.GetErrorRate(i) - classifier2.GetErrorRate(i);
}
double denominator = 0;
double numerator = 0;
for (var i = 0; i < classifier1.NumberOfExperiments() / 2; i++)
{
var mean = (difference[2 * i] + difference[2 * i + 1]) / 2;
numerator += mean;
var variance = (difference[2 * i] - mean) * (difference[2 * i] - mean) +
(difference[2 * i + 1] - mean) * (difference[2 * i + 1] - mean);
denominator += variance;
}
numerator = System.Math.Sqrt(10) * numerator / 5;
denominator = System.Math.Sqrt(denominator / 5);
return(numerator / denominator);
}
public override StatisticalTestResult Compare(ExperimentPerformance classifier1,
ExperimentPerformance classifier2)
{
var statistic = TestStatistic(classifier1, classifier2);
var degreeOfFreedom = classifier1.NumberOfExperiments() / 2;
return(new StatisticalTestResult(Distribution.TDistribution(statistic, degreeOfFreedom), false));
}
/**
* <summary> Execute Stratified K-fold cross-validation with the given classifier on the given data set using the given parameters.</summary>
*
* <param name="experiment">Experiment to be run.</param>
* <returns>An ExperimentPerformance instance.</returns>
*/
public override ExperimentPerformance Execute(Experiment experiment)
{
var result = new ExperimentPerformance();
var crossValidation =
new StratifiedKFoldCrossValidation <Instance.Instance>(experiment.GetDataSet().GetClassInstances(), K,
experiment.GetParameter().GetSeed());
RunExperiment(experiment.GetClassifier(), experiment.GetParameter(), result, crossValidation);
return(result);
}
protected void RunExperiment(Classifier.Classifier classifier, Parameter.Parameter parameter,
ExperimentPerformance experimentPerformance, CrossValidation <Instance.Instance> crossValidation,
InstanceList.InstanceList testSet)
{
for (var i = 0; i < K; i++)
{
var trainSet = new InstanceList.InstanceList(crossValidation.GetTrainFold(i));
classifier.Train(trainSet, parameter);
experimentPerformance.Add(classifier.Test(testSet));
}
}
/**
* Execute K-fold cross-validation with separate test set with the given classifier on the given data set using the given parameters.
*
* @param experiment Experiment to be run.
* @return An ExperimentPerformance instance.
*/
public override ExperimentPerformance Execute(Experiment experiment)
{
var result = new ExperimentPerformance();
var instanceList = experiment.GetDataSet().GetInstanceList();
var partition = instanceList.Partition(0.25, new Random(experiment.GetParameter().GetSeed()));
var crossValidation = new KFoldCrossValidation <Instance.Instance>(
partition.Get(1).GetInstances(),
K, experiment.GetParameter().GetSeed());
RunExperiment(experiment.GetClassifier(), experiment.GetParameter(), result, crossValidation,
partition.Get(0));
return(result);
}
/**
* <summary> Execute the bootstrap run with the given classifier on the given data set using the given parameters.</summary>
*
* <param name="experiment">Experiment to be run.</param>
* <returns>An ExperimentPerformance instance.</returns>
*/
public ExperimentPerformance Execute(Experiment experiment)
{
var result = new ExperimentPerformance();
for (var i = 0; i < _numberOfBootstraps; i++)
{
var bootstrap = new Bootstrap <Instance.Instance>(experiment.GetDataSet().GetInstances(),
i + experiment.GetParameter().GetSeed());
var bootstrapSample = new InstanceList.InstanceList(bootstrap.GetSample());
experiment.GetClassifier().Train(bootstrapSample, experiment.GetParameter());
result.Add(experiment.GetClassifier().Test(experiment.GetDataSet().GetInstanceList()));
}
return(result);
}
/**
* <summary> The execute method takes an {@link Experiment} and a {@link MultipleRun} as inputs. By selecting a candidateList from given
* Experiment it tries to find a FeatureSubSet that gives best performance.</summary>
*
* <param name="multipleRun">{@link MultipleRun} type input.</param>
* <param name="experiment"> {@link Experiment} type input.</param>
* <returns>FeatureSubSet that gives best performance.</returns>
*/
public FeatureSubSet Execute(MultipleRun multipleRun, Experiment.Experiment experiment)
{
var processed = new HashSet <FeatureSubSet>();
var best = initialSubSet;
processed.Add(best);
var betterFound = true;
ExperimentPerformance bestPerformance = null;
if (best.Size() > 0)
{
bestPerformance = multipleRun.Execute(experiment.FeatureSelectedExperiment(best));
}
while (betterFound)
{
betterFound = false;
var candidateList = Operator(best,
experiment.GetDataSet().GetDataDefinition().AttributeCount());
foreach (var candidateSubSet in candidateList)
{
if (!processed.Contains(candidateSubSet))
{
if (candidateSubSet.Size() > 0)
{
var currentPerformance =
multipleRun.Execute(experiment.FeatureSelectedExperiment(candidateSubSet));
if (bestPerformance == null || currentPerformance.IsBetter(bestPerformance))
{
best = candidateSubSet;
bestPerformance = currentPerformance;
betterFound = true;
}
}
processed.Add(candidateSubSet);
}
}
}
return(best);
}
public int Compare(ExperimentPerformance classifier1, ExperimentPerformance classifier2, double alpha)
{
var testResult1 = Compare(classifier1, classifier2);
var testResult2 = Compare(classifier2, classifier1);
var testResultType1 = testResult1.OneTailed(alpha);
var testResultType2 = testResult2.OneTailed(alpha);
if (testResultType1.Equals(StatisticalTestResultType.REJECT))
{
return(1);
}
if (testResultType2.Equals(StatisticalTestResultType.REJECT))
{
return(-1);
}
return(0);
}
private double TestStatistic(ExperimentPerformance classifier1, ExperimentPerformance classifier2)
{
var difference = new double[classifier1.NumberOfExperiments()];
var sum = 0.0;
for (var i = 0; i < classifier1.NumberOfExperiments(); i++)
{
difference[i] = classifier1.GetErrorRate(i) - classifier2.GetErrorRate(i);
sum += difference[i];
}
var mean = sum / classifier1.NumberOfExperiments();
sum = 0.0;
for (var i = 0; i < classifier1.NumberOfExperiments(); i++)
{
sum += (difference[i] - mean) * (difference[i] - mean);
}
var standardDeviation = System.Math.Sqrt(sum / (classifier1.NumberOfExperiments() - 1));
return(System.Math.Sqrt(classifier1.NumberOfExperiments()) * mean / standardDeviation);
}
public abstract StatisticalTestResult Compare(ExperimentPerformance classifier1,
ExperimentPerformance classifier2);
