private float GetAUROC(F64Matrix observations, double[] targets)
{
// print the raw data being used for classification
PrintArray(observations);
PrintVector(targets);
// split the data into training and test set
var splitter = new RandomTrainingTestIndexSplitter <double>(trainingPercentage: 0.5);
var trainingTestSplit = splitter.SplitSet(observations, targets);
var trainSet = trainingTestSplit.TrainingSet;
var testSet = trainingTestSplit.TestSet;
// train the model
var learner = new ClassificationRandomForestLearner();
var model = learner.Learn(trainSet.Observations, trainSet.Targets);
// make the predictions from the test set
var testPredictions = model.PredictProbability(testSet.Observations);
// create the metric and measure the error
var metric = new RocAucClassificationProbabilityMetric(1);
var testError = (float)metric.Error(testSet.Targets, testPredictions);
if (testError < .5f)
{
testError = 1f - testError;
}
return(testError);
}
public void RocAucClassificationMetric_Error_Not_Binary()
{
var targets = new double[] { 0, 1, 2 };
var probabilities = new ProbabilityPrediction[0];
var sut = new RocAucClassificationProbabilityMetric(1);
var actual = sut.Error(targets, probabilities);
}
public void RocAucClassificationMetric_Error()
{
var targets = new double[] { 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1 };
var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.052380952 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.993377483 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.111111111 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.193377483 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.793377483 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.012345679 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.885860173 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.714285714 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.985860173 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.985860173 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.993377483 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.993377483 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.954545455 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.020725389 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.985860173 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 0.985860173 }
}) };
var sut = new RocAucClassificationProbabilityMetric(1);
var actual = sut.Error(targets, probabilities);
Assert.AreEqual(0.0085470085470086277, actual, 0.00001);
}
public void RocAucClassificationMetric_Error_No_Error()
{
var targets = new double[] { 0, 1 };
var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0 }, { 1.0, 0.0 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 1 }
}) };
var sut = new RocAucClassificationProbabilityMetric(1);
var actual = sut.Error(targets, probabilities);
Assert.AreEqual(0.0, actual);
}
public void RocAucClassificationMetric_ErrorString()
{
var targets = new double[] { 0, 1 };
var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0 }, { 1.0, 0.0 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 1 }
}) };
var sut = new RocAucClassificationProbabilityMetric(1);
var actual = sut.ErrorString(targets, probabilities);
var expected = ";0;1;0;1\r\n0;1.000;0.000;100.000;0.000\r\n1;0.000;1.000;0.000;100.000\r\nError: 0.000\r\n";
Assert.AreEqual(expected, actual);
}
public void RocAucClassificationMetric_Error_Only_Negative_Targets()
{
var positives = Enumerable.Range(0, 10).Select(s => 0.0).ToList();
var targets = positives.ToArray();
var probabilities = targets
.Select(s => new ProbabilityPrediction(0.0, new Dictionary <double, double> {
{ 0.0, 1.0 }, { 1.0, 0.0 }
}))
.ToArray();
var sut = new RocAucClassificationProbabilityMetric(1);
sut.Error(targets, probabilities);
}
public void RocAucClassificationMetric_ErrorString_TargetStringMapping()
{
var targets = new double[] { 0, 1 };
var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
{ 0, 0 }, { 1.0, 0.0 }
}), new ProbabilityPrediction(1, new Dictionary <double, double> {
{ 0, 0.0 }, { 1.0, 1 }
}) };
var sut = new RocAucClassificationProbabilityMetric(1);
var targetStringMapping = new Dictionary <double, string> {
{ 0, "Negative" }, { 1, "Positive" }
};
var actual = sut.ErrorString(targets, probabilities, targetStringMapping);
var expected = ";Negative;Positive;Negative;Positive\r\nNegative;1.000;0.000;100.000;0.000\r\nPositive;0.000;1.000;0.000;100.000\r\nError: 0.000\r\n";
Assert.AreEqual(expected, actual);
}
public void RocAucClassificationMetric_Error_Always_Positve()
{
var positives = Enumerable.Range(0, 800).Select(s => 1.0).ToList();
var negatives = Enumerable.Range(0, 200).Select(s => 0.0).ToList();
positives.AddRange(negatives);
var targets = positives.ToArray();
var probabilities = targets
.Select(s => s == 0 ? new ProbabilityPrediction(0.0, new Dictionary <double, double> {
{ 0.0, 1.0 }, { 1.0, 1.0 }
}) :
new ProbabilityPrediction(1.0, new Dictionary <double, double> {
{ 0.0, 1.0 }, { 1.0, 1.0 }
}))
.ToArray();
var sut = new RocAucClassificationProbabilityMetric(1);
var actual = sut.Error(targets, probabilities);
Assert.AreEqual(0.5, actual, 0.0001);
}