/// <summary> /// Writes the evaluation results to a file with the specified name. /// </summary> /// <param name="writer">The name of the file to write the report to.</param> /// <param name="evaluator">The classifier evaluator.</param> /// <param name="x">The x vector of the ground truth.</param> /// <param name="y">The y of the ground truth.</param> /// <param name="yPredicDistrib">The predictive distributions.</param> /// <param name="yPredicLabel">The predicted labels.</param> /// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks> private void WriteReport( StreamWriter writer, ClassifierEvaluator <IList <Vector>, int, IList <string>, string> evaluator, Vector[] x, IList <string> y, IEnumerable <IDictionary <string, double> > yPredicDistrib, IEnumerable <string> yPredicLabel) { // Compute confusion matrix var confusionMatrix = evaluator.ConfusionMatrix(x, y, yPredicLabel); // Compute mean negative log probability double meanNegativeLogProbability = evaluator.Evaluate(x, y, yPredicDistrib, Metrics.NegativeLogProbability) / yPredicDistrib.Count(); // Compute M-measure (averaged pairwise AUC) IDictionary <string, IDictionary <string, double> > aucMatrix; double auc = evaluator.AreaUnderRocCurve(x, y, yPredicDistrib, out aucMatrix); // Compute per-label AUC as well as micro- and macro-averaged AUC double microAuc; double macroAuc; int macroAucClassLabelCount; var labelAuc = this.ComputeLabelAuc( confusionMatrix, evaluator, x, y, yPredicDistrib, out microAuc, out macroAuc, out macroAucClassLabelCount); // Instance-averaged performance this.WriteInstanceAveragedPerformance(writer, confusionMatrix, meanNegativeLogProbability, microAuc); // Class-averaged performance this.WriteClassAveragedPerformance(writer, confusionMatrix, auc, macroAuc, macroAucClassLabelCount); // Performance on individual classes this.WriteIndividualClassPerformance(writer, confusionMatrix, labelAuc); // Confusion matrix this.WriteConfusionMatrix(writer, confusionMatrix); // Pairwise AUC this.WriteAucMatrix(writer, aucMatrix); }
/// <summary> /// Writes the evaluation results to a file with the specified name. /// </summary> /// <param name="writer">The name of the file to write the report to.</param> /// <param name="evaluator">The classifier evaluator.</param> /// <param name="groundTruth">The ground truth.</param> /// <param name="predictiveDistributions">The predictive distributions.</param> /// <param name="predictedLabels">The predicted labels.</param> private void WriteReport( StreamWriter writer, ClassifierEvaluator <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string> evaluator, IList <LabeledFeatureValues> groundTruth, ICollection <IDictionary <string, double> > predictiveDistributions, IEnumerable <string> predictedLabels) { // Compute confusion matrix var confusionMatrix = evaluator.ConfusionMatrix(groundTruth, predictedLabels); // Compute mean negative log probability double meanNegativeLogProbability = evaluator.Evaluate(groundTruth, predictiveDistributions, Metrics.NegativeLogProbability) / predictiveDistributions.Count; // Compute M-measure (averaged pairwise AUC) IDictionary <string, IDictionary <string, double> > aucMatrix; double auc = evaluator.AreaUnderRocCurve(groundTruth, predictiveDistributions, out aucMatrix); // Compute per-label AUC as well as micro- and macro-averaged AUC double microAuc; double macroAuc; int macroAucClassLabelCount; var labelAuc = this.ComputeLabelAuc( confusionMatrix, evaluator, groundTruth, predictiveDistributions, out microAuc, out macroAuc, out macroAucClassLabelCount); // Instance-averaged performance this.WriteInstanceAveragedPerformance(writer, confusionMatrix, meanNegativeLogProbability, microAuc); // Class-averaged performance this.WriteClassAveragedPerformance(writer, confusionMatrix, auc, macroAuc, macroAucClassLabelCount); // Performance on individual classes this.WriteIndividualClassPerformance(writer, confusionMatrix, labelAuc); // Confusion matrix this.WriteConfusionMatrix(writer, confusionMatrix); // Pairwise AUC this.WriteAucMatrix(writer, aucMatrix); }