} /* GradeAgainstCrossValidationRun */
void CrossValidate(PicesFeatureVectorList testExamples,
PicesFeatureVectorList trainingExamples,
int foldNum
)
{
msgQueue.AddMsg("\n\n" + "Starting Fold[" + (foldNum + 1).ToString() + "] of Fold[" + numOfFolds + "]" + "\n");
if (trainingModel != null)
{
trainingModel = null;
}
msgQueue.AddMsg("Building Classifier");
try
{
trainingModel = new TrainingModel2(runLog, curSelModel);
trainingModel.BuildTrainingModel(trainingExamples);
}
catch (Exception e)
{
msgQueue.AddMsg("\n\n\n" + "Exception occur building training model" + "\n" + e.ToString() + "\n\n");
MessageBox.Show(this, "Exception Occurred Building Training Model." + "\n" + e.ToString(), "CrossValidate", MessageBoxButtons.OK);
trainingModel = null;
return;
}
if (cancelProcessing)
{
trainingModel = null;
return;
}
if (!trainingModel.Valid)
{
MessageBox.Show(this, "Training Model is Invalid.", "Ground Thruthing", MessageBoxButtons.OK);
}
else
{
msgQueue.AddMsg("\n" + "Predictions for fold[" + (foldNum + 1).ToString() + "]");
PicesClass knownClass = null;
foreach (PicesFeatureVector example in testExamples)
{
if (cancelProcessing)
{
break;
}
knownClass = example.MLClass;
PicesPrediction prediction = trainingModel.PredictClass(example);
if ((otherClass != null) && (prediction.MLClass == noAgreementClass))
{
prediction.MLClass = otherClass;
}
/**@todo Add code for "NoAgreement:" to be treeted as a Other Class when there is "NoAgreement". */
confusionMatrix.MakePrediction(knownClass, prediction.MLClass, prediction.Probability, example, trainingModel);
}
}
trainingModel.CleanUp();
trainingModel = null;
} /* CrossValidate */
} /* GradeAgainstGroundTruth */
private void GradeAgainstCrossValidationRun()
{
if (curSelModel == null)
{
curSelModel = new PicesTrainingConfigManaged(configFileName, runLog);
}
if (!curSelModel.Valid())
{
MessageBox.Show("Configuration File[" + configFileName + "] is not valid");
return;
}
bool changesMade = false;
PicesFeatureVectorList data = curSelModel.LoadFeatureDataFromTrainingLibraries(changesMade);
if (cancelProcessing)
{
data = null;
return;
}
if (data == null)
{
MessageBox.Show("No data was loaded from Configuration File[" + configFileName + "] is not valid");
return;
}
PicesFeatureVectorList examples = data.StratifyAmoungstClasses(numOfFolds);
int imageCount = examples.Count;
int numImagesPerFold = (imageCount + numOfFolds - 1) / numOfFolds;
int firstInGroup = 0;
int foldNum;
for (foldNum = 0; foldNum < numOfFolds; foldNum++)
{
int lastInGroup;
// If We are doing the last Fold Make sure that we are including all the examples
// that have not been tested.
if (foldNum == (numOfFolds - 1))
{
lastInGroup = imageCount;
}
else
{
lastInGroup = firstInGroup + numImagesPerFold - 1;
}
PicesFeatureVectorList trainingImages = new PicesFeatureVectorList();
PicesFeatureVectorList testImages = new PicesFeatureVectorList();
for (int x = 0; (x < imageCount) && (!cancelProcessing); x++)
{
PicesFeatureVector newImage = new PicesFeatureVector(examples[x]);
if ((x >= firstInGroup) && (x <= lastInGroup))
{
testImages.Add(newImage);
}
else
{
trainingImages.Add(newImage);
}
}
if (cancelProcessing)
{
break;
}
CrossValidate(testImages, trainingImages, foldNum);
firstInGroup = firstInGroup + numImagesPerFold;
}
if (!cancelProcessing)
{
if (trainingModel != null)
{
trainingModel = null;
}
msgQueue.AddMsg("\n" + "Building Final Classifier for User Query");
trainingModel = new TrainingModel2(runLog, curSelModel);
trainingModel.BuildTrainingModel(data);
}
if ((!cancelProcessing) && (trainingModel != null) && (!trainingModel.Valid))
{
MessageBox.Show("Error Loading Training Model; refer to dialog box");
cancelProcessing = true;
}
else if (!cancelProcessing)
{
confusionMatrix.SetTrainingModel(trainingModel);
}
// We no longer need the Training Model or Ground Truth loaded
BlockStart();
data = null;
BlockEnd();
} /* GradeAgainstCrossValidationRun */