/// <summary> /// Writes the probability calibration plot to the file with the specified name. /// </summary> /// <param name="fileName">The name of the file to write the calibration plot to.</param> /// <param name="evaluator">The classifier evaluator.</param> /// <param name="groundTruth">The ground truth.</param> /// <param name="predictiveDistributions">The predictive distributions.</param> /// <param name="positiveClassLabel">The label of the positive class.</param> private void WriteCalibrationCurve( string fileName, ClassifierEvaluator <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string> evaluator, IList <LabeledFeatureValues> groundTruth, IList <IDictionary <string, double> > predictiveDistributions, string positiveClassLabel) { Debug.Assert(predictiveDistributions != null, "The predictive distributions must not be null."); Debug.Assert(predictiveDistributions.Count > 0, "The predictive distributions must not be empty."); Debug.Assert(positiveClassLabel != null, "The label of the positive class must not be null."); var calibrationCurve = evaluator.CalibrationCurve(positiveClassLabel, groundTruth, predictiveDistributions); double calibrationError = calibrationCurve.Select(i => Metrics.AbsoluteError(i.EmpiricalProbability, i.PredictedProbability)).Average(); using (var writer = new StreamWriter(fileName)) { writer.WriteLine("# Empirical probability calibration plot"); writer.WriteLine("#"); writer.WriteLine("# Class '" + positiveClassLabel + "' (versus the rest)"); writer.WriteLine("# Calibration error = {0} (mean absolute error)", calibrationError); writer.WriteLine("#"); writer.WriteLine("# Predicted probability, empirical probability"); foreach (var point in calibrationCurve) { writer.WriteLine(point); } } }
/// <summary> /// Writes the probability calibration plot to the file with the specified name. /// </summary> /// <param name="fileName">The name of the file to write the calibration plot to.</param> /// <param name="evaluator">The classifier evaluator.</param> /// <param name="x">The x vector of the ground truth.</param> /// <param name="y">The y of the ground truth.</param> /// <param name="yPredicDistrib">The predictive distributions.</param> /// <param name="positiveClassLabel">The label of the positive class.</param> /// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks> private void WriteCalibrationCurve( string fileName, ClassifierEvaluator <IList <Vector>, int, IList <string>, string> evaluator, Vector[] x, IList <string> y, IEnumerable <IDictionary <string, double> > yPredicDistrib, string positiveClassLabel) { Debug.Assert(yPredicDistrib != null, "The predictive distributions must not be null."); Debug.Assert(yPredicDistrib.Count() > 0, "The predictive distributions must not be empty."); Debug.Assert(positiveClassLabel != null, "The label of the positive class must not be null."); var calibrationCurve = evaluator.CalibrationCurve(positiveClassLabel, x, y, yPredicDistrib); double calibrationError = calibrationCurve.Select(v => Metrics.AbsoluteError(v.First, v.Second)).Average(); using (var writer = new StreamWriter(fileName)) { writer.WriteLine("# Empirical probability calibration plot"); writer.WriteLine("#"); writer.WriteLine("# Class '" + positiveClassLabel + "' (versus the rest)"); writer.WriteLine("# Calibration error = {0} (mean absolute error)", calibrationError); writer.WriteLine("#"); writer.WriteLine("# Predicted probability, empirical probability"); foreach (var point in calibrationCurve) { writer.WriteLine("{0}, {1}", point.First, point.Second); } } }