private void EvaluateManyClassifiers(ClassifierParams[] classifierParamses, string targetTag, Func<ClassifierParams, string> debugInfo)
{
var problems = problemService.LoadAllDocumentsFromStorage();
TextDocument[] trainingSet;
TextDocument[] evaluationSet;
PrepareData(problems, out trainingSet, out evaluationSet, 60);
PrintStats("Training set: ", trainingSet, targetTag);
PrintStats("Evaluation set: ", evaluationSet, targetTag);
PrintDelimeter();
var bestScore = -1.0;
ClassifierParams bestParams = null;
foreach (var classifierParams in classifierParamses)
{
var evaluationResult = EvaluateSingle(trainingSet, classifierParams, TestsCount, 70, DebugLevel.No);
Console.WriteLine(debugInfo(classifierParams));
Console.WriteLine("FScore = {0}", evaluationResult.FScore);
if (evaluationResult.FScore > bestScore)
{
bestScore = evaluationResult.FScore;
bestParams = classifierParams;
}
}
PrintDelimeter();
Console.WriteLine("Best classifier");
Console.WriteLine(debugInfo(bestParams));
var result = EvaluateSingle(problems, bestParams, TestsCount, 60, DebugLevel.No);
Console.WriteLine("Evaluation FScore = {0}", result.FScore);
Console.WriteLine("Evaluation Precision = {0}", result.Precision);
Console.WriteLine("Evaluation Recall = {0}", result.Recall);
Console.WriteLine("Evaluation Accuracy = {0}", result.Accuracy);
}
private EvaluationResult EvaluateSingle(TextDocument[] set, ClassifierParams classifierParams, int testsCount, int trainingSetPercent, DebugLevel debugLevel)
{
return EvaluationResult.Combine(Enumerable.Range(0, testsCount).Select(i =>
{
TextDocument[] trainingSet;
TextDocument[] evaluationSet;
PrepareData(set, out trainingSet, out evaluationSet, trainingSetPercent);
var result = classifierBuilder.Build(trainingSet, classifierParams).Classifier;
EvaluateInner(result, classifierParams, trainingSet, debugLevel);
return EvaluateInner(result, classifierParams, evaluationSet, debugLevel);
}));
}
private EvaluationResult EvaluateInner(IClassifier classifier, ClassifierParams classifierParams, TextDocument[] evaluationSet, DebugLevel debugLevel)
{
var evaluationResult = classifierEvaluator.Evaluate(classifier, evaluationSet, classifierParams.TargetTag);
if (debugLevel == DebugLevel.FullWithDump)
{
PrintDelimeter();
}
if (debugLevel >= DebugLevel.OnlyFScore)
{
Console.WriteLine("Evaluation FScore = {0}", evaluationResult.FScore);
}
if (debugLevel >= DebugLevel.Full)
{
Console.WriteLine("Evaluation Precision = {0}", evaluationResult.Precision);
Console.WriteLine("Evaluation Recall = {0}", evaluationResult.Recall);
Console.WriteLine("Evaluation Accuracy = {0}", evaluationResult.Accuracy);
}
if (debugLevel >= DebugLevel.FullWithDump)
{
DumpEvaluation(evaluationResult);
}
return evaluationResult;
}
