예제 #1
0
    private float GetAUROC(F64Matrix observations, double[] targets)
    {
        // print the raw data being used for classification
        PrintArray(observations);
        PrintVector(targets);

        // split the data into training and test set
        var splitter          = new RandomTrainingTestIndexSplitter <double>(trainingPercentage: 0.5);
        var trainingTestSplit = splitter.SplitSet(observations, targets);
        var trainSet          = trainingTestSplit.TrainingSet;
        var testSet           = trainingTestSplit.TestSet;

        // train the model
        var learner = new ClassificationRandomForestLearner();
        var model   = learner.Learn(trainSet.Observations, trainSet.Targets);

        // make the predictions from the test set
        var testPredictions = model.PredictProbability(testSet.Observations);

        // create the metric and measure the error
        var metric    = new RocAucClassificationProbabilityMetric(1);
        var testError = (float)metric.Error(testSet.Targets, testPredictions);

        if (testError < .5f)
        {
            testError = 1f - testError;
        }

        return(testError);
    }
        public void RocAucClassificationMetric_Error_Not_Binary()
        {
            var targets       = new double[] { 0, 1, 2 };
            var probabilities = new ProbabilityPrediction[0];

            var sut    = new RocAucClassificationProbabilityMetric(1);
            var actual = sut.Error(targets, probabilities);
        }
        public void RocAucClassificationMetric_Error()
        {
            var targets       = new double[] { 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1 };
            var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.052380952 }
                }), new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.020725389 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.993377483 }
                }), new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.020725389 }
                }), new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.020725389 }
                }), new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.111111111 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.193377483 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.793377483 }
                }), new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.020725389 }
                }), new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.012345679 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.885860173 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.714285714 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.985860173 }
                }), new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.020725389 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.985860173 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.993377483 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.993377483 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.954545455 }
                }), new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.020725389 }
                }), new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.020725389 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.985860173 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 0.985860173 }
                }) };

            var sut    = new RocAucClassificationProbabilityMetric(1);
            var actual = sut.Error(targets, probabilities);

            Assert.AreEqual(0.0085470085470086277, actual, 0.00001);
        }
        public void RocAucClassificationMetric_Error_No_Error()
        {
            var targets       = new double[] { 0, 1 };
            var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0 }, { 1.0, 0.0 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 1 }
                }) };
            var sut    = new RocAucClassificationProbabilityMetric(1);
            var actual = sut.Error(targets, probabilities);

            Assert.AreEqual(0.0, actual);
        }
        public void RocAucClassificationMetric_ErrorString()
        {
            var targets       = new double[] { 0, 1 };
            var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0 }, { 1.0, 0.0 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 1 }
                }) };
            var sut    = new RocAucClassificationProbabilityMetric(1);
            var actual = sut.ErrorString(targets, probabilities);

            var expected = ";0;1;0;1\r\n0;1.000;0.000;100.000;0.000\r\n1;0.000;1.000;0.000;100.000\r\nError: 0.000\r\n";

            Assert.AreEqual(expected, actual);
        }
        public void RocAucClassificationMetric_Error_Only_Negative_Targets()
        {
            var positives = Enumerable.Range(0, 10).Select(s => 0.0).ToList();
            var targets   = positives.ToArray();

            var probabilities = targets
                                .Select(s => new ProbabilityPrediction(0.0, new Dictionary <double, double> {
                { 0.0, 1.0 }, { 1.0, 0.0 }
            }))
                                .ToArray();

            var sut = new RocAucClassificationProbabilityMetric(1);

            sut.Error(targets, probabilities);
        }
        public void RocAucClassificationMetric_ErrorString_TargetStringMapping()
        {
            var targets       = new double[] { 0, 1 };
            var probabilities = new ProbabilityPrediction[] { new ProbabilityPrediction(0, new Dictionary <double, double> {
                    { 0, 0 }, { 1.0, 0.0 }
                }), new ProbabilityPrediction(1, new Dictionary <double, double> {
                    { 0, 0.0 }, { 1.0, 1 }
                }) };
            var sut = new RocAucClassificationProbabilityMetric(1);
            var targetStringMapping = new Dictionary <double, string> {
                { 0, "Negative" }, { 1, "Positive" }
            };

            var actual   = sut.ErrorString(targets, probabilities, targetStringMapping);
            var expected = ";Negative;Positive;Negative;Positive\r\nNegative;1.000;0.000;100.000;0.000\r\nPositive;0.000;1.000;0.000;100.000\r\nError: 0.000\r\n";

            Assert.AreEqual(expected, actual);
        }
        public void RocAucClassificationMetric_Error_Always_Positve()
        {
            var positives = Enumerable.Range(0, 800).Select(s => 1.0).ToList();
            var negatives = Enumerable.Range(0, 200).Select(s => 0.0).ToList();

            positives.AddRange(negatives);
            var targets = positives.ToArray();

            var probabilities = targets
                                .Select(s => s == 0 ? new ProbabilityPrediction(0.0, new Dictionary <double, double> {
                { 0.0, 1.0 }, { 1.0, 1.0 }
            }) :
                                        new ProbabilityPrediction(1.0, new Dictionary <double, double> {
                { 0.0, 1.0 }, { 1.0, 1.0 }
            }))
                                .ToArray();

            var sut    = new RocAucClassificationProbabilityMetric(1);
            var actual = sut.Error(targets, probabilities);

            Assert.AreEqual(0.5, actual, 0.0001);
        }