public static void Main(string[] args)
{
Evaluation evaluation = new Evaluation();
evaluation.addEvaluator(new ErrorEvaluator(new MeanSquaredError()));
string[] classNames = new string[] { "Virginica", "Setosa", "Versicolor" };
MultiLayerPerceptron neuralNet = (MultiLayerPerceptron)NeuralNetwork.createFromFile("irisNet.nnet");
DataSet dataSet = DataSet.createFromFile("data_sets/iris_data_normalised.txt", 4, 3, ",");
evaluation.addEvaluator(new ClassifierEvaluator.MultiClass(classNames));
evaluation.evaluateDataSet(neuralNet, dataSet);
ClassifierEvaluator evaluator = evaluation.getEvaluator(typeof(ClassifierEvaluator.MultiClass));
ConfusionMatrix confusionMatrix = evaluator.Result;
Console.WriteLine("Confusion matrrix:\r\n");
Console.WriteLine(confusionMatrix.ToString() + "\r\n\r\n");
Console.WriteLine("Classification metrics\r\n");
ClassificationMetrics[] metrics = ClassificationMetrics.createFromMatrix(confusionMatrix);
ClassificationMetrics.Stats average = ClassificationMetrics.average(metrics);
foreach (ClassificationMetrics cm in metrics)
{
Console.WriteLine(cm.ToString() + "\r\n");
}
Console.WriteLine(average.ToString());
}
/// <summary>
/// Writes the precision-recall curve to the file with the specified name.
/// </summary>
/// <param name="fileName">The name of the file to write the precision-recall curve to.</param>
/// <param name="evaluator">The classifier evaluator.</param>
/// <param name="x">The x vector of the ground truth.</param>
/// <param name="y">The y of the ground truth.</param>
/// <param name="yPredicDistrib">The predictive distributions.</param>
/// <param name="positiveClassLabel">The label of the positive class.</param>
/// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks>
private void WritePrecisionRecallCurve(
string fileName,
ClassifierEvaluator <IList <Vector>, int, IList <string>, string> evaluator,
Vector[] x,
IList <string> y,
IEnumerable <IDictionary <string, double> > yPredicDistrib,
string positiveClassLabel)
{
Debug.Assert(yPredicDistrib != null, "The predictive distributions must not be null.");
Debug.Assert(yPredicDistrib.Count() > 0, "The predictive distributions must not be empty.");
Debug.Assert(positiveClassLabel != null, "The label of the positive class must not be null.");
var precisionRecallCurve = evaluator.PrecisionRecallCurve(positiveClassLabel, x, y, yPredicDistrib);
using (var writer = new StreamWriter(fileName))
{
writer.WriteLine("# Precision-recall curve");
writer.WriteLine("#");
writer.WriteLine("# Class '" + positiveClassLabel + "' (versus the rest)");
writer.WriteLine("#");
writer.WriteLine("# Recall (R), precision (P)");
foreach (var point in precisionRecallCurve)
{
writer.WriteLine("{0}, {1}", point.First, point.Second);
}
}
}
/// <summary>
/// Writes the receiver operating characteristic curve to the file with the specified name.
/// </summary>
/// <param name="fileName">The name of the file to write the receiver operating characteristic curve to.</param>
/// <param name="evaluator">The classifier evaluator.</param>
/// <param name="groundTruth">The ground truth.</param>
/// <param name="predictiveDistributions">The predictive distributions.</param>
/// <param name="positiveClassLabel">The label of the positive class.</param>
private void WriteRocCurve(
string fileName,
ClassifierEvaluator <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string> evaluator,
IList <LabeledFeatureValues> groundTruth,
IList <IDictionary <string, double> > predictiveDistributions,
string positiveClassLabel)
{
Debug.Assert(predictiveDistributions != null, "The predictive distributions must not be null.");
Debug.Assert(predictiveDistributions.Count > 0, "The predictive distributions must not be empty.");
Debug.Assert(positiveClassLabel != null, "The label of the positive class must not be null.");
var rocCurve = evaluator.ReceiverOperatingCharacteristicCurve(positiveClassLabel, groundTruth, predictiveDistributions);
using (var writer = new StreamWriter(fileName))
{
writer.WriteLine("# Receiver operating characteristic (ROC) curve");
writer.WriteLine("#");
writer.WriteLine("# Class '" + positiveClassLabel + "' (versus the rest)");
writer.WriteLine("#");
writer.WriteLine("# False positive rate (FPR), True positive rate (TPR)");
foreach (var point in rocCurve)
{
writer.WriteLine(point);
}
}
}
public BPM(
string[] labels,
double sparsityApproxThresh)
{
Debug.Assert(labels != null, "The labels must not be null.");
Debug.Assert(labels.Length == 2, "The labels must have two possible values.");
Debug.Assert(sparsityApproxThresh >= 0, "The sparsityApproxThresh must be greater than or equal to zero.");
// Initialise the validations
_validate = new Validate();
// Initialise the BPM
_engine = new Dictionary <DistributionType, InferenceEngine>();
_w = new Dictionary <DistributionType, Variable <Vector> >();
_w[DistributionType.Prior] = null;
_w[DistributionType.Posterior] = null;
_d = new Dictionary <DistributionType, DistributionName>();
_yPredicDistrib = Enumerable.Empty <IDictionary <string, double> >();
_yPredicLabel = new string[] { };
_mapping = new GenericClassifierMapping(labels);
// TO DO
// Evaluator mapping
var evaluatorMapping = _mapping.ForEvaluation();
_evaluator = new ClassifierEvaluator
<IList <Vector>, int, IList <string>, string>(evaluatorMapping);
// Other initialisations
_availableDatasetName = new DatasetName();
_numObservations = 0;
_numFeatures = 0;
}
/// <summary>
/// Writes the probability calibration plot to the file with the specified name.
/// </summary>
/// <param name="fileName">The name of the file to write the calibration plot to.</param>
/// <param name="evaluator">The classifier evaluator.</param>
/// <param name="groundTruth">The ground truth.</param>
/// <param name="predictiveDistributions">The predictive distributions.</param>
/// <param name="positiveClassLabel">The label of the positive class.</param>
private void WriteCalibrationCurve(
string fileName,
ClassifierEvaluator <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string> evaluator,
IList <LabeledFeatureValues> groundTruth,
IList <IDictionary <string, double> > predictiveDistributions,
string positiveClassLabel)
{
Debug.Assert(predictiveDistributions != null, "The predictive distributions must not be null.");
Debug.Assert(predictiveDistributions.Count > 0, "The predictive distributions must not be empty.");
Debug.Assert(positiveClassLabel != null, "The label of the positive class must not be null.");
var calibrationCurve = evaluator.CalibrationCurve(positiveClassLabel, groundTruth, predictiveDistributions);
double calibrationError = calibrationCurve.Select(i => Metrics.AbsoluteError(i.EmpiricalProbability, i.PredictedProbability)).Average();
using (var writer = new StreamWriter(fileName))
{
writer.WriteLine("# Empirical probability calibration plot");
writer.WriteLine("#");
writer.WriteLine("# Class '" + positiveClassLabel + "' (versus the rest)");
writer.WriteLine("# Calibration error = {0} (mean absolute error)", calibrationError);
writer.WriteLine("#");
writer.WriteLine("# Predicted probability, empirical probability");
foreach (var point in calibrationCurve)
{
writer.WriteLine(point);
}
}
}
/// <summary>
/// Writes the precision-recall curve to the file with the specified name.
/// </summary>
/// <param name="fileName">The name of the file to write the precision-recall curve to.</param>
/// <param name="evaluator">The classifier evaluator.</param>
/// <param name="groundTruth">The ground truth.</param>
/// <param name="predictiveDistributions">The predictive distributions.</param>
/// <param name="positiveClassLabel">The label of the positive class.</param>
private void WritePrecisionRecallCurve(
string fileName,
ClassifierEvaluator <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string> evaluator,
IList <LabeledFeatureValues> groundTruth,
IList <IDictionary <string, double> > predictiveDistributions,
string positiveClassLabel)
{
Debug.Assert(predictiveDistributions != null, "The predictive distributions must not be null.");
Debug.Assert(predictiveDistributions.Count > 0, "The predictive distributions must not be empty.");
Debug.Assert(positiveClassLabel != null, "The label of the positive class must not be null.");
var precisionRecallCurve = evaluator.PrecisionRecallCurve(positiveClassLabel, groundTruth, predictiveDistributions);
using (var writer = new StreamWriter(fileName))
{
writer.WriteLine("# Precision-recall curve");
writer.WriteLine("#");
writer.WriteLine("# Class '" + positiveClassLabel + "' (versus the rest)");
writer.WriteLine("#");
writer.WriteLine("# precision (P), Recall (R)");
foreach (var point in precisionRecallCurve)
{
writer.WriteLine(point);
}
}
}
/// <summary>
/// Writes the receiver operating characteristic curve to the file with the specified name.
/// </summary>
/// <param name="fileName">The name of the file to write the receiver operating characteristic curve to.</param>
/// <param name="evaluator">The classifier evaluator.</param>
/// <param name="x">The x vector of the ground truth.</param>
/// <param name="y">The y of the ground truth.</param>
/// <param name="yPredicDistrib">The predictive distributions.</param>
/// <param name="positiveClassLabel">The label of the positive class.</param>
/// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks>
private void WriteRocCurve(
string fileName,
ClassifierEvaluator <IList <Vector>, int, IList <string>, string> evaluator,
Vector[] x,
IList <string> y,
IEnumerable <IDictionary <string, double> > yPredicDistrib,
string positiveClassLabel)
{
Debug.Assert(yPredicDistrib != null, "The predictive distributions must not be null.");
Debug.Assert(yPredicDistrib.Count() > 0, "The predictive distributions must not be empty.");
Debug.Assert(positiveClassLabel != null, "The label of the positive class must not be null.");
var rocCurve = evaluator.ReceiverOperatingCharacteristicCurve(positiveClassLabel, x, y, yPredicDistrib);
using (var writer = new StreamWriter(fileName))
{
writer.WriteLine("# Receiver operating characteristic (ROC) curve");
writer.WriteLine("#");
writer.WriteLine("# Class '" + positiveClassLabel + "' (versus the rest)");
writer.WriteLine("#");
writer.WriteLine("# False positive rate (FPR), true positive rate (TPR)");
foreach (var point in rocCurve)
{
writer.WriteLine("{0}, {1}", point.First, point.Second);
}
}
}
/// <summary>
/// Writes the probability calibration plot to the file with the specified name.
/// </summary>
/// <param name="fileName">The name of the file to write the calibration plot to.</param>
/// <param name="evaluator">The classifier evaluator.</param>
/// <param name="x">The x vector of the ground truth.</param>
/// <param name="y">The y of the ground truth.</param>
/// <param name="yPredicDistrib">The predictive distributions.</param>
/// <param name="positiveClassLabel">The label of the positive class.</param>
/// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks>
private void WriteCalibrationCurve(
string fileName,
ClassifierEvaluator <IList <Vector>, int, IList <string>, string> evaluator,
Vector[] x,
IList <string> y,
IEnumerable <IDictionary <string, double> > yPredicDistrib,
string positiveClassLabel)
{
Debug.Assert(yPredicDistrib != null, "The predictive distributions must not be null.");
Debug.Assert(yPredicDistrib.Count() > 0, "The predictive distributions must not be empty.");
Debug.Assert(positiveClassLabel != null, "The label of the positive class must not be null.");
var calibrationCurve = evaluator.CalibrationCurve(positiveClassLabel, x, y, yPredicDistrib);
double calibrationError = calibrationCurve.Select(v => Metrics.AbsoluteError(v.First, v.Second)).Average();
using (var writer = new StreamWriter(fileName))
{
writer.WriteLine("# Empirical probability calibration plot");
writer.WriteLine("#");
writer.WriteLine("# Class '" + positiveClassLabel + "' (versus the rest)");
writer.WriteLine("# Calibration error = {0} (mean absolute error)", calibrationError);
writer.WriteLine("#");
writer.WriteLine("# Predicted probability, empirical probability");
foreach (var point in calibrationCurve)
{
writer.WriteLine("{0}, {1}", point.First, point.Second);
}
}
}
// TODO: dont sysout - store somewhere these results so they can be displayed
//
private void testNetwork(NeuralNetwork neuralNetwork, DataSet testSet)
{
evaluation.evaluateDataSet(neuralNetwork, testSet);
// works for binary what if we have multiple classes - how to get results for multiple classes here?
// results.add(evaluation.getEvaluator(ClassificationMetricsEvaluator.class).getResult()[0]); // MUST BE FIXED!!!!! get all and add thm all to results
System.Console.WriteLine("##############################################################################");
System.Console.WriteLine("MeanSquare Error: " + ((ErrorEvaluator)evaluation.getEvaluator(typeof(ErrorEvaluator))).Result);
System.Console.WriteLine("##############################################################################");
// TODO: deal with BinaryClassifiers too here
ClassifierEvaluator evaluator = ((ClassifierEvaluator.MultiClass)evaluation.getEvaluator(typeof(ClassifierEvaluator.MultiClass)));
ConfusionMatrix confusionMatrix = evaluator.Result;
System.Console.WriteLine("Confusion Matrix: \r\n" + confusionMatrix.ToString());
System.Console.WriteLine("##############################################################################");
System.Console.WriteLine("Classification metrics: ");
ClassificationMetrics[] metrics = ClassificationMetrics.createFromMatrix(confusionMatrix); // add all of these to result
foreach (ClassificationMetrics cm in metrics)
{
System.Console.WriteLine(cm.ToString());
}
System.Console.WriteLine("##############################################################################");
}
/// <summary>
/// Computes all per-label AUCs as well as the micro- and macro-averaged AUCs.
/// </summary>
/// <param name="confusionMatrix">The confusion matrix.</param>
/// <param name="evaluator">The classifier evaluator.</param>
/// <param name="x">The x vector of the ground truth.</param>
/// <param name="y">The y of the ground truth.</param>
/// <param name="yPredicDistrib">The predictive distributions.</param>
/// <param name="microAuc">The micro-averaged area under the receiver operating characteristic curve.</param>
/// <param name="macroAuc">The macro-averaged area under the receiver operating characteristic curve.</param>
/// <param name="macroAucClassLabelCount">The number of class labels for which the AUC if defined.</param>
/// <returns>The area under the receiver operating characteristic curve for each class label.</returns>
/// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks>
private IDictionary <string, double> ComputeLabelAuc(
ConfusionMatrix <string> confusionMatrix,
ClassifierEvaluator <IList <Vector>, int, IList <string>, string> evaluator,
Vector[] x,
IList <string> y,
IEnumerable <IDictionary <string, double> > yPredicDistrib,
out double microAuc,
out double macroAuc,
out int macroAucClassLabelCount)
{
int instanceCount = yPredicDistrib.Count();
var classLabels = confusionMatrix.ClassLabelSet.Elements.ToArray();
int classLabelCount = classLabels.Length;
var labelAuc = new Dictionary <string, double>();
// Compute per-label AUC
macroAucClassLabelCount = classLabelCount;
foreach (var classLabel in classLabels)
{
// One versus rest
double auc;
try
{
auc = evaluator.AreaUnderRocCurve(classLabel, x, y, yPredicDistrib);
}
catch (ArgumentException)
{
auc = double.NaN;
macroAucClassLabelCount--;
}
labelAuc.Add(classLabel, auc);
}
// Compute micro- and macro-averaged AUC
microAuc = 0;
macroAuc = 0;
foreach (var label in classLabels)
{
if (double.IsNaN(labelAuc[label]))
{
continue;
}
microAuc += confusionMatrix.TrueLabelCount(label) * labelAuc[label] / instanceCount;
macroAuc += labelAuc[label] / macroAucClassLabelCount;
}
return(labelAuc);
}
/// <summary>
/// Computes all per-label AUCs as well as the micro- and macro-averaged AUCs.
/// </summary>
/// <param name="confusionMatrix">The confusion matrix.</param>
/// <param name="evaluator">The classifier evaluator.</param>
/// <param name="groundTruth">The ground truth.</param>
/// <param name="predictiveDistributions">The predictive distributions.</param>
/// <param name="microAuc">The micro-averaged area under the receiver operating characteristic curve.</param>
/// <param name="macroAuc">The macro-averaged area under the receiver operating characteristic curve.</param>
/// <param name="macroAucClassLabelCount">The number of class labels for which the AUC if defined.</param>
/// <returns>The area under the receiver operating characteristic curve for each class label.</returns>
private IDictionary <string, double> ComputeLabelAuc(
ConfusionMatrix <string> confusionMatrix,
ClassifierEvaluator <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string> evaluator,
IList <LabeledFeatureValues> groundTruth,
ICollection <IDictionary <string, double> > predictiveDistributions,
out double microAuc,
out double macroAuc,
out int macroAucClassLabelCount)
{
int instanceCount = predictiveDistributions.Count;
var classLabels = confusionMatrix.ClassLabelSet.Elements.ToArray();
int classLabelCount = classLabels.Length;
var labelAuc = new Dictionary <string, double>();
// Compute per-label AUC
macroAucClassLabelCount = classLabelCount;
foreach (var classLabel in classLabels)
{
// One versus rest
double auc;
try
{
auc = evaluator.AreaUnderRocCurve(classLabel, groundTruth, predictiveDistributions);
}
catch (ArgumentException)
{
auc = double.NaN;
macroAucClassLabelCount--;
}
labelAuc.Add(classLabel, auc);
}
// Compute micro- and macro-averaged AUC
microAuc = 0;
macroAuc = 0;
foreach (var label in classLabels)
{
if (double.IsNaN(labelAuc[label]))
{
continue;
}
microAuc += confusionMatrix.TrueLabelCount(label) * labelAuc[label] / instanceCount;
macroAuc += labelAuc[label] / macroAucClassLabelCount;
}
return(labelAuc);
}
/// <summary>
/// Writes the evaluation results to a file with the specified name.
/// </summary>
/// <param name="writer">The name of the file to write the report to.</param>
/// <param name="evaluator">The classifier evaluator.</param>
/// <param name="x">The x vector of the ground truth.</param>
/// <param name="y">The y of the ground truth.</param>
/// <param name="yPredicDistrib">The predictive distributions.</param>
/// <param name="yPredicLabel">The predicted labels.</param>
/// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks>
private void WriteReport(
StreamWriter writer,
ClassifierEvaluator <IList <Vector>, int, IList <string>, string> evaluator,
Vector[] x,
IList <string> y,
IEnumerable <IDictionary <string, double> > yPredicDistrib,
IEnumerable <string> yPredicLabel)
{
// Compute confusion matrix
var confusionMatrix = evaluator.ConfusionMatrix(x, y, yPredicLabel);
// Compute mean negative log probability
double meanNegativeLogProbability =
evaluator.Evaluate(x, y, yPredicDistrib, Metrics.NegativeLogProbability) / yPredicDistrib.Count();
// Compute M-measure (averaged pairwise AUC)
IDictionary <string, IDictionary <string, double> > aucMatrix;
double auc = evaluator.AreaUnderRocCurve(x, y, yPredicDistrib, out aucMatrix);
// Compute per-label AUC as well as micro- and macro-averaged AUC
double microAuc;
double macroAuc;
int macroAucClassLabelCount;
var labelAuc = this.ComputeLabelAuc(
confusionMatrix,
evaluator,
x,
y,
yPredicDistrib,
out microAuc,
out macroAuc,
out macroAucClassLabelCount);
// Instance-averaged performance
this.WriteInstanceAveragedPerformance(writer, confusionMatrix, meanNegativeLogProbability, microAuc);
// Class-averaged performance
this.WriteClassAveragedPerformance(writer, confusionMatrix, auc, macroAuc, macroAucClassLabelCount);
// Performance on individual classes
this.WriteIndividualClassPerformance(writer, confusionMatrix, labelAuc);
// Confusion matrix
this.WriteConfusionMatrix(writer, confusionMatrix);
// Pairwise AUC
this.WriteAucMatrix(writer, aucMatrix);
}
/// <summary>
/// Writes the evaluation results to a file with the specified name.
/// </summary>
/// <param name="writer">The name of the file to write the report to.</param>
/// <param name="evaluator">The classifier evaluator.</param>
/// <param name="groundTruth">The ground truth.</param>
/// <param name="predictiveDistributions">The predictive distributions.</param>
/// <param name="predictedLabels">The predicted labels.</param>
private void WriteReport(
StreamWriter writer,
ClassifierEvaluator <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string> evaluator,
IList <LabeledFeatureValues> groundTruth,
ICollection <IDictionary <string, double> > predictiveDistributions,
IEnumerable <string> predictedLabels)
{
// Compute confusion matrix
var confusionMatrix = evaluator.ConfusionMatrix(groundTruth, predictedLabels);
// Compute mean negative log probability
double meanNegativeLogProbability =
evaluator.Evaluate(groundTruth, predictiveDistributions, Metrics.NegativeLogProbability) / predictiveDistributions.Count;
// Compute M-measure (averaged pairwise AUC)
IDictionary <string, IDictionary <string, double> > aucMatrix;
double auc = evaluator.AreaUnderRocCurve(groundTruth, predictiveDistributions, out aucMatrix);
// Compute per-label AUC as well as micro- and macro-averaged AUC
double microAuc;
double macroAuc;
int macroAucClassLabelCount;
var labelAuc = this.ComputeLabelAuc(
confusionMatrix,
evaluator,
groundTruth,
predictiveDistributions,
out microAuc,
out macroAuc,
out macroAucClassLabelCount);
// Instance-averaged performance
this.WriteInstanceAveragedPerformance(writer, confusionMatrix, meanNegativeLogProbability, microAuc);
// Class-averaged performance
this.WriteClassAveragedPerformance(writer, confusionMatrix, auc, macroAuc, macroAucClassLabelCount);
// Performance on individual classes
this.WriteIndividualClassPerformance(writer, confusionMatrix, labelAuc);
// Confusion matrix
this.WriteConfusionMatrix(writer, confusionMatrix);
// Pairwise AUC
this.WriteAucMatrix(writer, aucMatrix);
}
/// <summary>
/// Prepares the environment before each test.
/// </summary>
public ClassifierEvaluatorTests()
{
// Ground truth labels (no uncertainty)
this.groundTruth = new LabelDistribution[5];
this.groundTruth[0] = new Dictionary <string, double> {
{ LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 }
};
this.groundTruth[1] = new Dictionary <string, double> {
{ LabelSet[0], 0 }, { LabelSet[1], 1 }, { LabelSet[2], 0 }
};
this.groundTruth[2] = new Dictionary <string, double> {
{ LabelSet[0], 0 }, { LabelSet[1], 0 }, { LabelSet[2], 1 }
};
this.groundTruth[3] = new Dictionary <string, double> {
{ LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 }
};
this.groundTruth[4] = new Dictionary <string, double> {
{ LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 }
};
// Predictions
this.predictions = new LabelDistribution[5];
this.predictions[0] = new Dictionary <string, double> {
{ LabelSet[0], 0 }, { LabelSet[1], 0 }, { LabelSet[2], 1 }
};
this.predictions[1] = new Dictionary <string, double> {
{ LabelSet[0], 0 }, { LabelSet[1], 1 }, { LabelSet[2], 0 }
};
this.predictions[2] = new Dictionary <string, double> {
{ LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 }
};
this.predictions[3] = new Dictionary <string, double> {
{ LabelSet[0], 1 / 6.0 }, { LabelSet[1], 2 / 3.0 }, { LabelSet[2], 1 / 6.0 }
};
this.predictions[4] = new Dictionary <string, double> {
{ LabelSet[0], 1 / 8.0 }, { LabelSet[1], 1 / 8.0 }, { LabelSet[2], 3 / 4.0 }
};
// Classifier evaluator
var classifierMapping = new ClassifierMapping();
var evaluatorMapping = classifierMapping.ForEvaluation();
this.evaluator = new ClassifierEvaluator <IEnumerable <LabelDistribution>, LabelDistribution, IEnumerable <LabelDistribution>, string>(evaluatorMapping);
}
public BPMMapped(
string[] labels)
{
Debug.Assert(labels != null, "The labels must not be null.");
Debug.Assert(labels.Length == 2, "The labels must have two possible values.");
// Initialise the validations
_validate = new Validate();
// Create a BPM from the mapping
_mapping = new GenericClassifierMapping(labels);
_classifier = BayesPointMachineClassifier.CreateBinaryClassifier(_mapping);
// Evaluator mapping
var evaluatorMapping = _mapping.ForEvaluation();
_evaluator = new ClassifierEvaluator
<IList <Vector>, int, IList <string>, string>(evaluatorMapping);
// Other initialisations
_availableDatasetName = new DatasetName();
_numObservations = 0;
_numFeatures = 0;
}
/// <summary>
/// Runs the module.
/// </summary>
/// <param name="args">The command line arguments for the module.</param>
/// <param name="usagePrefix">The prefix to print before the usage string.</param>
/// <returns>True if the run was successful, false otherwise.</returns>
public override bool Run(string[] args, string usagePrefix)
{
string groundTruthFileName = string.Empty;
string predictionsFileName = string.Empty;
string reportFileName = string.Empty;
string calibrationCurveFileName = string.Empty;
string rocCurveFileName = string.Empty;
string precisionRecallCurveFileName = string.Empty;
string positiveClassLabel = string.Empty;
var parser = new CommandLineParser();
parser.RegisterParameterHandler("--ground-truth", "FILE", "File with ground truth labels", v => groundTruthFileName = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--predictions", "FILE", "File with label predictions", v => predictionsFileName = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--report", "FILE", "File to store the evaluation report", v => reportFileName = v, CommandLineParameterType.Optional);
parser.RegisterParameterHandler("--calibration-curve", "FILE", "File to store the empirical calibration curve", v => calibrationCurveFileName = v, CommandLineParameterType.Optional);
parser.RegisterParameterHandler("--roc-curve", "FILE", "File to store the receiver operating characteristic curve", v => rocCurveFileName = v, CommandLineParameterType.Optional);
parser.RegisterParameterHandler("--precision-recall-curve", "FILE", "File to store the precision-recall curve", v => precisionRecallCurveFileName = v, CommandLineParameterType.Optional);
parser.RegisterParameterHandler("--positive-class", "STRING", "Label of the positive class to use in curves", v => positiveClassLabel = v, CommandLineParameterType.Optional);
if (!parser.TryParse(args, usagePrefix))
{
return(false);
}
// Read ground truth
var groundTruth = ClassifierPersistenceUtils.LoadLabeledFeatureValues(groundTruthFileName);
// Read predictions using ground truth label dictionary
var predictions = ClassifierPersistenceUtils.LoadLabelDistributions(predictionsFileName, groundTruth.First().LabelDistribution.LabelSet);
// Check that there are at least two distinct class labels
if (predictions.First().LabelSet.Count < 2)
{
throw new InvalidFileFormatException("Ground truth and predictions must contain at least two distinct class labels.");
}
// Distill distributions and point estimates
var predictiveDistributions = predictions.Select(i => i.ToDictionary()).ToList();
var predictivePointEstimates = predictions.Select(i => i.GetMode()).ToList();
// Create evaluator
var evaluatorMapping = Mappings.Classifier.ForEvaluation();
var evaluator = new ClassifierEvaluator <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string>(evaluatorMapping);
// Write evaluation report
if (!string.IsNullOrEmpty(reportFileName))
{
using (var writer = new StreamWriter(reportFileName))
{
this.WriteReportHeader(writer, groundTruthFileName, predictionsFileName);
this.WriteReport(writer, evaluator, groundTruth, predictiveDistributions, predictivePointEstimates);
}
}
// Compute and write the empirical probability calibration curve
positiveClassLabel = this.CheckPositiveClassLabel(groundTruth, positiveClassLabel);
if (!string.IsNullOrEmpty(calibrationCurveFileName))
{
this.WriteCalibrationCurve(calibrationCurveFileName, evaluator, groundTruth, predictiveDistributions, positiveClassLabel);
}
// Compute and write the precision-recall curve
if (!string.IsNullOrEmpty(precisionRecallCurveFileName))
{
this.WritePrecisionRecallCurve(precisionRecallCurveFileName, evaluator, groundTruth, predictiveDistributions, positiveClassLabel);
}
// Compute and write the receiver operating characteristic curve
if (!string.IsNullOrEmpty(rocCurveFileName))
{
this.WriteRocCurve(rocCurveFileName, evaluator, groundTruth, predictiveDistributions, positiveClassLabel);
}
return(true);
}
/// <summary>
/// Runs the module.
/// </summary>
/// <param name="args">The command line arguments for the module.</param>
/// <param name="usagePrefix">The prefix to print before the usage string.</param>
/// <returns>True if the run was successful, false otherwise.</returns>
public override bool Run(string[] args, string usagePrefix)
{
string dataSetFile = string.Empty;
string resultsFile = string.Empty;
int crossValidationFoldCount = 5;
int iterationCount = BayesPointMachineClassifierTrainingSettings.IterationCountDefault;
int batchCount = BayesPointMachineClassifierTrainingSettings.BatchCountDefault;
bool computeModelEvidence = BayesPointMachineClassifierTrainingSettings.ComputeModelEvidenceDefault;
var parser = new CommandLineParser();
parser.RegisterParameterHandler("--data-set", "FILE", "File with training data", v => dataSetFile = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--results", "FILE", "File with cross-validation results", v => resultsFile = v, CommandLineParameterType.Required);
parser.RegisterParameterHandler("--folds", "NUM", "Number of cross-validation folds (defaults to " + crossValidationFoldCount + ")", v => crossValidationFoldCount = v, CommandLineParameterType.Optional);
parser.RegisterParameterHandler("--iterations", "NUM", "Number of training algorithm iterations (defaults to " + iterationCount + ")", v => iterationCount = v, CommandLineParameterType.Optional);
parser.RegisterParameterHandler("--batches", "NUM", "Number of batches to split the training data into (defaults to " + batchCount + ")", v => batchCount = v, CommandLineParameterType.Optional);
parser.RegisterParameterHandler("--compute-evidence", "Compute model evidence (defaults to " + computeModelEvidence + ")", () => computeModelEvidence = true);
if (!parser.TryParse(args, usagePrefix))
{
return(false);
}
// Load and shuffle data
var dataSet = ClassifierPersistenceUtils.LoadLabeledFeatureValues(dataSetFile);
BayesPointMachineClassifierModuleUtilities.WriteDataSetInfo(dataSet);
Rand.Restart(562);
Rand.Shuffle(dataSet);
// Create evaluator
var evaluatorMapping = Mappings.Classifier.ForEvaluation();
var evaluator = new ClassifierEvaluator <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string>(evaluatorMapping);
// Create performance metrics
var accuracy = new List <double>();
var negativeLogProbability = new List <double>();
var auc = new List <double>();
var evidence = new List <double>();
var iterationCounts = new List <double>();
var trainingTime = new List <double>();
// Run cross-validation
int validationSetSize = dataSet.Count / crossValidationFoldCount;
Console.WriteLine("Running {0}-fold cross-validation on {1}", crossValidationFoldCount, dataSetFile);
// TODO: Use chained mapping to implement cross-validation
for (int fold = 0; fold < crossValidationFoldCount; fold++)
{
// Construct training and validation sets for fold
int validationSetStart = fold * validationSetSize;
int validationSetEnd = (fold + 1 == crossValidationFoldCount)
? dataSet.Count
: (fold + 1) * validationSetSize;
var trainingSet = new List <LabeledFeatureValues>();
var validationSet = new List <LabeledFeatureValues>();
for (int instance = 0; instance < dataSet.Count; instance++)
{
if (validationSetStart <= instance && instance < validationSetEnd)
{
validationSet.Add(dataSet[instance]);
}
else
{
trainingSet.Add(dataSet[instance]);
}
}
// Print info
Console.WriteLine(" Fold {0} [validation set instances {1} - {2}]", fold + 1, validationSetStart, validationSetEnd - 1);
// Create classifier
var classifier = BayesPointMachineClassifier.CreateBinaryClassifier(Mappings.Classifier);
classifier.Settings.Training.IterationCount = iterationCount;
classifier.Settings.Training.BatchCount = batchCount;
classifier.Settings.Training.ComputeModelEvidence = computeModelEvidence;
int currentIterationCount = 0;
classifier.IterationChanged += (sender, eventArgs) => { currentIterationCount = eventArgs.CompletedIterationCount; };
// Train classifier
var stopWatch = new Stopwatch();
stopWatch.Start();
classifier.Train(trainingSet);
stopWatch.Stop();
// Produce predictions
var predictions = classifier.PredictDistribution(validationSet).ToList();
var predictedLabels = predictions.Select(
prediction => prediction.Aggregate((aggregate, next) => next.Value > aggregate.Value ? next : aggregate).Key).ToList();
// Iteration count
iterationCounts.Add(currentIterationCount);
// Training time
trainingTime.Add(stopWatch.ElapsedMilliseconds);
// Compute accuracy
accuracy.Add(1 - (evaluator.Evaluate(validationSet, predictedLabels, Metrics.ZeroOneError) / predictions.Count));
// Compute mean negative log probability
negativeLogProbability.Add(evaluator.Evaluate(validationSet, predictions, Metrics.NegativeLogProbability) / predictions.Count);
// Compute M-measure (averaged pairwise AUC)
auc.Add(evaluator.AreaUnderRocCurve(validationSet, predictions));
// Compute log evidence if desired
evidence.Add(computeModelEvidence ? classifier.LogModelEvidence : double.NaN);
// Persist performance metrics
Console.WriteLine(
" Accuracy = {0,5:0.0000} NegLogProb = {1,5:0.0000} AUC = {2,5:0.0000}{3} Iterations = {4} Training time = {5}",
accuracy[fold],
negativeLogProbability[fold],
auc[fold],
computeModelEvidence ? string.Format(" Log evidence = {0,5:0.0000}", evidence[fold]) : string.Empty,
iterationCounts[fold],
BayesPointMachineClassifierModuleUtilities.FormatElapsedTime(trainingTime[fold]));
BayesPointMachineClassifierModuleUtilities.SavePerformanceMetrics(
resultsFile, accuracy, negativeLogProbability, auc, evidence, iterationCounts, trainingTime);
}
return(true);
}
/// <summary>
/// CrossValidate diagnosis
/// </summary>
/// <param name="x"></param>
/// <param name="y"></param>
/// <param name="mapping"></param>
/// <param name="reportFileName"></param>
/// <param name="crossValidationFoldCount"></param>
/// <param name="iterationCount"></param>
/// <param name="computeModelEvidence"></param>
/// <param name="batchCount"></param>
/// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks>
public CrossValidateMapped(
Vector[] x,
IList <string> y,
GenericClassifierMapping mapping,
string reportFileName,
int crossValidationFoldCount, //folds
int iterationCount,
bool computeModelEvidence,
int batchCount)
{
Debug.Assert(x != null, "The feature vector must not be null.");
Debug.Assert(y != null, "The targe variable must not be null.");
Debug.Assert(mapping != null, "The classifier map must not be null.");
Debug.Assert(!string.IsNullOrEmpty(reportFileName), "The report file name must not be null/empty.");
Debug.Assert(iterationCount > 0, "The iteration count must be greater than zero.");
Debug.Assert(batchCount > 0, "The batch count must be greater than zero.");
// Shuffle dataset
shuffleVector(x);
// Create evaluator
var evaluatorMapping = mapping.ForEvaluation();
var evaluator = new ClassifierEvaluator <
IList <Vector>, // the type of the instance source,
int, // the type of an instance
IList <string>, // the type of the label source
string>( // the type of a label.
evaluatorMapping);
// Create performance metrics
var accuracy = new List <double>();
var negativeLogProbability = new List <double>();
var auc = new List <double>();
var evidence = new List <double>();
var iterationCounts = new List <double>();
var trainingTime = new List <double>();
// Run cross-validation
int validationSetSize = x.Length / crossValidationFoldCount;
int trainingSetSize = x.Length - validationSetSize;
int validationFoldSetSize = 0;
int trainingFoldSetSize = 0;
Console.WriteLine(
"Running {0}-fold cross-validation", crossValidationFoldCount);
if (validationSetSize == 0 || trainingSetSize == 0)
{
Console.WriteLine("Invalid number of folds");
Console.ReadKey();
System.Environment.Exit(1);
}
for (int fold = 0; fold < crossValidationFoldCount; fold++)
{
// Construct training and validation sets for fold
int validationSetStart = fold * validationSetSize;
int validationSetEnd = (fold + 1 == crossValidationFoldCount)
? x.Length
: (fold + 1) * validationSetSize;
validationFoldSetSize = validationSetEnd - validationSetStart;
trainingFoldSetSize = x.Length - validationFoldSetSize;
Vector[] trainingSet = new Vector[trainingFoldSetSize];
Vector[] validationSet = new Vector[validationFoldSetSize];
IList <string> trainingSetLabels = new List <string>();
IList <string> validationSetLabels = new List <string>();
for (int instance = 0, iv = 0, it = 0; instance < x.Length; instance++)
{
if (validationSetStart <= instance && instance < validationSetEnd)
{
validationSet[iv++] = x[instance];
validationSetLabels.Add(y[instance]);
}
else
{
trainingSet[it++] = x[instance];
trainingSetLabels.Add(y[instance]);
}
}
// Print info
Console.WriteLine(" Fold {0} [validation set instances {1} - {2}]", fold + 1, validationSetStart, validationSetEnd - 1);
// Create classifier
var classifier = BayesPointMachineClassifier.CreateBinaryClassifier(mapping);
classifier.Settings.Training.IterationCount = iterationCount;
classifier.Settings.Training.BatchCount = batchCount;
classifier.Settings.Training.ComputeModelEvidence = computeModelEvidence;
int currentIterationCount = 0;
classifier.IterationChanged += (sender, eventArgs) => { currentIterationCount = eventArgs.CompletedIterationCount; };
// Train classifier
var stopWatch = new Stopwatch();
stopWatch.Start();
classifier.Train(trainingSet, trainingSetLabels);
stopWatch.Stop();
// Produce predictions
IEnumerable <IDictionary <string, double> > predictions =
classifier.PredictDistribution(validationSet);
var predictedLabels = classifier.Predict(validationSet);
// Iteration count
iterationCounts.Add(currentIterationCount);
// Training time
trainingTime.Add(stopWatch.ElapsedMilliseconds);
// Compute accuracy
accuracy.Add(1 - (evaluator.Evaluate(validationSet, validationSetLabels, predictedLabels, Metrics.ZeroOneError) / predictions.Count()));
// Compute mean negative log probability
negativeLogProbability.Add(evaluator.Evaluate(validationSet, validationSetLabels, predictions, Metrics.NegativeLogProbability) / predictions.Count());
// Compute M-measure (averaged pairwise AUC)
auc.Add(evaluator.AreaUnderRocCurve(validationSet, validationSetLabels, predictions));
// Compute log evidence if desired
evidence.Add(computeModelEvidence ? classifier.LogModelEvidence : double.NaN);
// Persist performance metrics
Console.WriteLine(
" Accuracy = {0,5:0.0000} NegLogProb = {1,5:0.0000} AUC = {2,5:0.0000}{3} Iterations = {4} Training time = {5}",
accuracy[fold],
negativeLogProbability[fold],
auc[fold],
computeModelEvidence ? string.Format(" Log evidence = {0,5:0.0000}", evidence[fold]) : string.Empty,
iterationCounts[fold],
FormatElapsedTime(trainingTime[fold]));
SavePerformanceMetrics(
reportFileName, accuracy, negativeLogProbability, auc, evidence, iterationCounts, trainingTime);
}
}
/// <summary>
///
/// </summary>
/// <param name="evaluator"></param>
/// <param name="x"></param>
/// <param name="y"></param>
/// <param name="yPredicDistrib"></param>
/// <param name="yPredicLabel"></param>
/// <param name="reportFileName"></param>
/// <param name="positiveClassLabel"></param>
/// <param name="calibrationCurveFileName"></param>
/// <param name="rocCurveFileName"></param>
/// <param name="groundTruthFileName"></param>
/// <param name="predictionsFileName"></param>
/// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks>
public EvaluationReportsMapped(
IBayesPointMachineClassifier <
IList <Vector>,
int,
IList <string>,
string,
IDictionary <string, double>,
BayesPointMachineClassifierTrainingSettings,
BinaryBayesPointMachineClassifierPredictionSettings <string> > classifier,
ClassifierEvaluator <IList <Vector>, int, IList <string>, string> evaluator,
Vector[] x,
IList <string> y,
IEnumerable <IDictionary <string, double> > yPredicDistrib,
IEnumerable <string> yPredicLabel,
string reportFileName,
string positiveClassLabel,
string groundTruthFileName = "",
string predictionsFileName = "",
string weightsFileName = "",
string calibrationCurveFileName = "",
string precisionRecallCurveFileName = "",
string rocCurveFileName = "")
{
Debug.Assert(classifier != null, "The classifier must not be null.");
Debug.Assert(evaluator != null, "The evaluator must not be null.");
Debug.Assert(x != null, "The feature vector must not be null.");
Debug.Assert(y != null, "The targe variable must not be null.");
Debug.Assert(yPredicDistrib != null, "The predictive distribution must not be null.");
Debug.Assert(yPredicLabel != null, "The predicted labels must not be null.");
Debug.Assert(!string.IsNullOrEmpty(reportFileName), "The report file name must not be null/empty.");
Debug.Assert(!string.IsNullOrEmpty(positiveClassLabel), "The positive class label must not be null/empty.");
// Write evaluation report header information
if (!string.IsNullOrEmpty(reportFileName))
{
using (var writer = new StreamWriter(reportFileName))
{
this.WriteReportHeader(writer, groundTruthFileName, predictionsFileName);
this.WriteReport(writer, evaluator, x, y, yPredicDistrib, yPredicLabel);
}
}
// Write the prediction distribution for all labels
if (!string.IsNullOrEmpty(predictionsFileName))
{
SaveLabelDistributions(predictionsFileName, yPredicDistrib);
}
// Compute and write the empirical probability calibration curve
if (!string.IsNullOrEmpty(calibrationCurveFileName))
{
this.WriteCalibrationCurve(calibrationCurveFileName, evaluator, x, y, yPredicDistrib, positiveClassLabel);
}
// Compute and write the precision-recall curve
if (!string.IsNullOrEmpty(precisionRecallCurveFileName))
{
this.WritePrecisionRecallCurve(precisionRecallCurveFileName, evaluator, x, y, yPredicDistrib, positiveClassLabel);
}
// Compute and write the receiver operating characteristic curve
if (!string.IsNullOrEmpty(rocCurveFileName))
{
this.WriteRocCurve(rocCurveFileName, evaluator, x, y, yPredicDistrib, positiveClassLabel);
}
// Compute and write the weights
if (!string.IsNullOrEmpty(weightsFileName))
{
this.SampleWeights(weightsFileName, classifier);
}
}
