public void ClassificationModel_PredictProbability_Threshold_On_Probability()
{
#region learner creation
// Use StreamReader(filepath) when running from filesystem
var parser = new CsvParser(() => new StringReader(Resources.winequality_white));
var targetName = "quality";
// read feature matrix
var observations = parser.EnumerateRows(c => c != targetName)
.ToF64Matrix();
// read classification targets and convert to binary problem (low quality/high quality).
var targets = parser.EnumerateRows(targetName)
.ToF64Vector().Select(t => t < 5 ? 0.0 : 1.0).ToArray();
var translation = new Dictionary <double, string> {
{ 0.0, "Low quality" }, { 1.0, "High quality" }
};
// create learner
var learner = new ClassificationDecisionTreeLearner(maximumTreeDepth: 5);
#endregion
// learns a ClassificationDecisionTreeModel
var model = learner.Learn(observations, targets);
// predict probabilities for all observations
var probabilityPredictions = model.PredictProbability(observations);
// zip target and probabilities to keep order
var zip = targets.Zip(probabilityPredictions, (t, p) => new { Target = t, Prediction = p });
// threhold on the probabilty of the predicted class.
// This will remove the obserations that the model is uncertain about.
var probabilityThreshold = 0.90;
var thresholdedResult = zip.Where(kvp => kvp.Prediction.Probabilities[kvp.Prediction.Prediction] > probabilityThreshold);
// evaluate the resulting observations
var thresholdedPredictions = thresholdedResult.Select(p => p.Prediction).ToArray();
var thresholdedTargets = thresholdedResult.Select(p => p.Target).ToArray();
// evaluate only on probability thresholded data
var metric = new LogLossClassificationProbabilityMetric();
Trace.WriteLine("ProbabilityThresholded Result:");
Trace.WriteLine(metric.ErrorString(thresholdedTargets, thresholdedPredictions, translation));
Trace.WriteLine("");
// evaluate on all data for comparison
Trace.WriteLine("All data result:");
Trace.WriteLine(metric.ErrorString(targets, probabilityPredictions, translation));
}
public void LogLossClassificationMetric_ErrorString_TargetStringMapping()
{
var sut = new LogLossClassificationProbabilityMetric(1e-15);
var predictions = new ProbabilityPrediction[] {
new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 1.0 }, { 1, 1.0 }, { 2, 1.0 }
}),
new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 1.0 }, { 2, 0.0 }
}),
new ProbabilityPrediction(2, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 }
}),
};
var targets = new double[] { 0, 1, 2 };
var targetStringMapping = new Dictionary <double, string> {
{ 0, "One" }, { 1, "Two" }, { 2, "Three" }
};
var actual = sut.ErrorString(targets, predictions, targetStringMapping);
var expected = ";One;Two;Three;One;Two;Three\r\nOne;1.000;0.000;0.000;100.000;0.000;0.000\r\nTwo;0.000;1.000;0.000;0.000;100.000;0.000\r\nThree;0.000;0.000;1.000;0.000;0.000;100.000\r\nError: 36.620\r\n";
Assert.AreEqual(expected, actual);
}
public void LogLossClassificationMetric_ErrorString()
{
var sut = new LogLossClassificationProbabilityMetric(1e-15);
var predictions = new ProbabilityPrediction[] {
new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 1.0 }, { 1, 1.0 }, { 2, 1.0 }
}),
new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 1.0 }, { 2, 0.0 }
}),
new ProbabilityPrediction(2, new Dictionary <double, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 1.0 }
}),
};
var targets = new double[] { 0, 1, 2 };
var actual = sut.ErrorString(targets, predictions);
var expected = ";0;1;2;0;1;2\r\n0;1.000;0.000;0.000;100.000;0.000;0.000\r\n1;0.000;1.000;0.000;0.000;100.000;0.000\r\n2;0.000;0.000;1.000;0.000;0.000;100.000\r\nError: 36.620\r\n";
Assert.AreEqual(expected, actual);
}
