protected void RunFold(int foldN)
{
var foldProfile = new CrossValidationTimeProfile {
FoldN = foldN, FoldStartTime = DateTime.Now
};
var foldProfiles = new ConcurrentDictionary <string, CrossValidationTimeProfile>();
foldProfiles.TryAdd("", foldProfile);
mFoldModelTimes.TryAdd(foldN, foldProfiles);
// fold data
LabeledDataset <LblT, InputExT> testSet, trainSet;
if (IsStratified)
{
Dataset.SplitForStratifiedCrossValidation(NumFolds, foldN, out trainSet, out testSet);
}
else
{
Dataset.SplitForCrossValidation(NumFolds, foldN, out trainSet, out testSet);
}
BeforeFold(foldN, trainSet, testSet);
// pefrorm mapping
ILabeledDataset <LblT, ModelExT> mappedTrainSet = MapTrainSet(foldN, trainSet);
ILabeledDataset <LblT, ModelExT> mappedTestSet = MapTestSet(foldN, testSet);
// validate
foreach (IModel <LblT, ModelExT> model in Models)
{
string modelName = GetModelName(model);
var modelProfile = new CrossValidationTimeProfile
{
FoldN = foldN,
ModelName = modelName,
FoldStartTime = foldProfile.FoldStartTime,
TrainStartTime = DateTime.Now
};
foldProfiles.TryAdd(modelName, modelProfile);
DoRunModel(foldN, model, trainSet, mappedTrainSet, testSet, mappedTestSet, modelProfile);
}
foldProfile.FoldEndTime = DateTime.Now;
AfterFold(foldN, trainSet, testSet);
}
protected void RunModel(int foldN, IModel <LblT, ModelExT> model,
ILabeledDataset <LblT, InputExT> trainSet, ILabeledDataset <LblT, ModelExT> mappedTrainSet,
ILabeledDataset <LblT, InputExT> testSet, ILabeledDataset <LblT, ModelExT> mappedTestSet, CrossValidationTimeProfile modelProfile)
{
// train
ILabeledDataset <LblT, ModelExT> usedTrainSet = BeforeTrain(foldN, model, trainSet, mappedTrainSet);
Train(foldN, model, usedTrainSet);
AfterTrain(foldN, model, usedTrainSet);
modelProfile.TrainEndTime = DateTime.Now;
// test
modelProfile.TestStartTime = DateTime.Now;
ILabeledDataset <LblT, ModelExT> usedTestSet = BeforeTest(foldN, model, testSet, mappedTestSet);
PerfMatrix <LblT> foldMatrix = GetPerfMatrix(GetModelName(model), foldN);
for (int i = 0; i < usedTestSet.Count; i++)
{
LabeledExample <LblT, ModelExT> le = usedTestSet[i];
Prediction <LblT> prediction = Predict(foldN, model, le);
if (AfterPrediction(foldN, model, testSet[i].Example, le, prediction) && prediction.Any())
{
foldMatrix.AddCount(le.Label, prediction.BestClassLabel);
}
}
modelProfile.TestEndTime = DateTime.Now;
AfterTest(foldN, model, usedTestSet);
}
protected override void DoRunModel(int foldN, IModel <LblT, ModelExT> model,
ILabeledDataset <LblT, InputExT> trainSet, ILabeledDataset <LblT, ModelExT> mappedTrainSet,
ILabeledDataset <LblT, InputExT> testSet, ILabeledDataset <LblT, ModelExT> mappedTestSet, CrossValidationTimeProfile modelProfile)
{
RunModel(foldN, model, trainSet, mappedTrainSet, testSet, mappedTestSet, modelProfile);
}
protected abstract void DoRunModel(int foldN, IModel <LblT, ModelExT> model,
ILabeledDataset <LblT, InputExT> trainSet, ILabeledDataset <LblT, ModelExT> mappedTrainSet,
ILabeledDataset <LblT, InputExT> testSet, ILabeledDataset <LblT, ModelExT> mappedTestSet, CrossValidationTimeProfile modelProfile);
protected override void DoRunModel(int foldN, IModel <LblT, ModelExT> model,
ILabeledDataset <LblT, InputExT> trainSet, ILabeledDataset <LblT, ModelExT> mappedTrainSet,
ILabeledDataset <LblT, InputExT> testSet, ILabeledDataset <LblT, ModelExT> mappedTestSet, CrossValidationTimeProfile modelProfile)
{
if (mFoldTasks != null)
{
RunModel(foldN, model, trainSet, mappedTrainSet, testSet, mappedTestSet, modelProfile);
}
else
{
lock (mFoldModelTasks[foldN])
{
mFoldModelTasks[foldN].Add(() => RunModel(foldN, model, trainSet, mappedTrainSet, testSet, mappedTestSet, modelProfile));
}
}
}