private void EvaluateManyClassifiers(ClassifierParams[] classifierParamses, string targetTag, Func<ClassifierParams, string> debugInfo) { var problems = problemService.LoadAllDocumentsFromStorage(); TextDocument[] trainingSet; TextDocument[] evaluationSet; PrepareData(problems, out trainingSet, out evaluationSet, 60); PrintStats("Training set: ", trainingSet, targetTag); PrintStats("Evaluation set: ", evaluationSet, targetTag); PrintDelimeter(); var bestScore = -1.0; ClassifierParams bestParams = null; foreach (var classifierParams in classifierParamses) { var evaluationResult = EvaluateSingle(trainingSet, classifierParams, TestsCount, 70, DebugLevel.No); Console.WriteLine(debugInfo(classifierParams)); Console.WriteLine("FScore = {0}", evaluationResult.FScore); if (evaluationResult.FScore > bestScore) { bestScore = evaluationResult.FScore; bestParams = classifierParams; } } PrintDelimeter(); Console.WriteLine("Best classifier"); Console.WriteLine(debugInfo(bestParams)); var result = EvaluateSingle(problems, bestParams, TestsCount, 60, DebugLevel.No); Console.WriteLine("Evaluation FScore = {0}", result.FScore); Console.WriteLine("Evaluation Precision = {0}", result.Precision); Console.WriteLine("Evaluation Recall = {0}", result.Recall); Console.WriteLine("Evaluation Accuracy = {0}", result.Accuracy); }
private EvaluationResult EvaluateSingle(TextDocument[] set, ClassifierParams classifierParams, int testsCount, int trainingSetPercent, DebugLevel debugLevel) { return EvaluationResult.Combine(Enumerable.Range(0, testsCount).Select(i => { TextDocument[] trainingSet; TextDocument[] evaluationSet; PrepareData(set, out trainingSet, out evaluationSet, trainingSetPercent); var result = classifierBuilder.Build(trainingSet, classifierParams).Classifier; EvaluateInner(result, classifierParams, trainingSet, debugLevel); return EvaluateInner(result, classifierParams, evaluationSet, debugLevel); })); }
private EvaluationResult EvaluateInner(IClassifier classifier, ClassifierParams classifierParams, TextDocument[] evaluationSet, DebugLevel debugLevel) { var evaluationResult = classifierEvaluator.Evaluate(classifier, evaluationSet, classifierParams.TargetTag); if (debugLevel == DebugLevel.FullWithDump) { PrintDelimeter(); } if (debugLevel >= DebugLevel.OnlyFScore) { Console.WriteLine("Evaluation FScore = {0}", evaluationResult.FScore); } if (debugLevel >= DebugLevel.Full) { Console.WriteLine("Evaluation Precision = {0}", evaluationResult.Precision); Console.WriteLine("Evaluation Recall = {0}", evaluationResult.Recall); Console.WriteLine("Evaluation Accuracy = {0}", evaluationResult.Accuracy); } if (debugLevel >= DebugLevel.FullWithDump) { DumpEvaluation(evaluationResult); } return evaluationResult; }