public void TestIncorrectFractionSums()
 {
     Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0.5, 0, 0.1, 0, false));
     Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0.6, 0, 0, 0, false));
     Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0, 0.5, 0, 0.6, false));
 }
Exemplo n.º 2
0
        public void TestPrecisionRecallCurve()
        {
            // General
            var expectedCurve = new[]
            {
                new PrecisionRecall(1, 0),
                new PrecisionRecall(0, 0),
                new PrecisionRecall(0.5, 0.5),
                new PrecisionRecall(2 / (double)3, 1),
                new PrecisionRecall(0.5, 1)
            };
            var computedCurve = Metrics.PrecisionRecallCurve(new[] { 1, 2 }, new Dictionary <int, double> {
                { 3, 1 }, { 1, 0.5 }, { 2, 0.25 }, { 4, 0 }
            }).ToArray();

            Assert.Equal(expectedCurve.Length, computedCurve.Length);
            for (int i = 0; i < expectedCurve.Length; i++)
            {
                Assert.Equal(expectedCurve[i], computedCurve[i]);
            }

            // No instance scores
            expectedCurve = new[] { new PrecisionRecall(1, 0), };
            computedCurve = Metrics.PrecisionRecallCurve(new[] { 1 }, new Dictionary <int, double>()).ToArray();
            Assert.Equal(expectedCurve.Length, computedCurve.Length);
            for (int i = 0; i < expectedCurve.Length; i++)
            {
                Assert.Equal(expectedCurve[i], computedCurve[i]);
            }

            // No negative instance scores
            expectedCurve = new[] { new PrecisionRecall(1, 0), new PrecisionRecall(1, 1) };
            computedCurve = Metrics.PrecisionRecallCurve(new[] { 1 }, new Dictionary <int, double> {
                { 1, 1 }
            }).ToArray();
            Assert.Equal(expectedCurve.Length, computedCurve.Length);
            for (int i = 0; i < expectedCurve.Length; i++)
            {
                Assert.Equal(expectedCurve[i], computedCurve[i]);
            }

            // Duplicate positive instances
            computedCurve = Metrics.PrecisionRecallCurve(new[] { 1, 1 }, new Dictionary <int, double> {
                { 1, 1 }
            }).ToArray();
            Assert.Equal(expectedCurve.Length, computedCurve.Length);
            for (int i = 0; i < expectedCurve.Length; i++)
            {
                Assert.Equal(expectedCurve[i], computedCurve[i]);
            }

            // No positive instance scores
            Assert.Throws <ArgumentException>(() => Metrics.PrecisionRecallCurve(new int[] { }, new Dictionary <int, double> {
                { 1, 1 }
            }));

            // Null checks
            Assert.Throws <ArgumentNullException>(() => Metrics.PrecisionRecallCurve(null, new Dictionary <int, double> {
                { 1, 1 }
            }));
            Assert.Throws <ArgumentNullException>(() => Metrics.PrecisionRecallCurve(new[] { 1 }, null));
        }
        public void AreaUnderRocCurveTest()
        {
            // AUC for perfect predictions

            // Per-label AUC
            Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[0], this.groundTruth, this.groundTruth));
            Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[1], this.groundTruth, this.groundTruth));
            Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[2], this.groundTruth, this.groundTruth));

            // M-measure
            IDictionary <string, IDictionary <string, double> > computedAucMatrix;

            Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.groundTruth, out computedAucMatrix));
            Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.groundTruth));

            // Pairwise AUC (upper triangle)
            Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]);
            Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[2]]);
            Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[2]]);

            // Pairwise AUC (diagnonal)
            foreach (string label in LabelSet)
            {
                Assert.Equal(double.NaN, computedAucMatrix[label][label]); // undefined result
            }

            // Pairwise AUC (lower triangle)
            Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]);
            Assert.Equal(1.0, computedAucMatrix[LabelSet[2]][LabelSet[0]]);
            Assert.Equal(1.0, computedAucMatrix[LabelSet[2]][LabelSet[1]]);

            // AUC for imperfect predictions

            // Per-label AUC
            Assert.Equal(5 / 12.0, this.evaluator.AreaUnderRocCurve(LabelSet[0], this.groundTruth, this.predictions)); // matches ROC curve
            Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[1], this.groundTruth, this.predictions));
            Assert.Equal(1 / 8.0, this.evaluator.AreaUnderRocCurve(LabelSet[2], this.groundTruth, this.predictions));

            // M-measure
            Assert.Equal(5 / 9.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.predictions, out computedAucMatrix));
            Assert.Equal(5 / 9.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.predictions));

            // Pairwise AUC (upper triangle)
            Assert.Equal(5 / 6.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]); // matches ROC curve
            Assert.Equal(0.0, computedAucMatrix[LabelSet[0]][LabelSet[2]]);
            Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[2]]);

            // Pairwise AUC (diagnonal)
            foreach (string label in LabelSet)
            {
                Assert.Equal(double.NaN, computedAucMatrix[label][label]); // undefined result
            }

            // Pairwise AUC (lower triangle)
            Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]);
            Assert.Equal(0.0, computedAucMatrix[LabelSet[2]][LabelSet[0]]);
            Assert.Equal(0.5, computedAucMatrix[LabelSet[2]][LabelSet[1]]);

            // Test code path for symmetric two-class case
            var binaryGroundTruth = new LabelDistribution[2];

            binaryGroundTruth[0] = new Dictionary <string, double> {
                { LabelSet[0], 1 }, { LabelSet[1], 0 }
            };
            binaryGroundTruth[1] = new Dictionary <string, double> {
                { LabelSet[0], 0 }, { LabelSet[1], 1 }
            };

            var binaryPredictions = new LabelDistribution[2];

            binaryPredictions[0] = new Dictionary <string, double> {
                { LabelSet[0], 0.9 }, { LabelSet[1], 0.1 }
            };
            binaryPredictions[1] = new Dictionary <string, double> {
                { LabelSet[0], 0.8 }, { LabelSet[1], 0.2 }
            };

            Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(binaryGroundTruth, binaryPredictions, out computedAucMatrix));
            Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(binaryGroundTruth, binaryPredictions));
            Assert.Equal(double.NaN, computedAucMatrix[LabelSet[0]][LabelSet[0]]); // undefined result
            Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]);
            Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]);
            Assert.Equal(double.NaN, computedAucMatrix[LabelSet[1]][LabelSet[1]]); // undefined result

            // No positive or negative class labels
            var actualLabelDistribution = new LabelDistribution[1];

            actualLabelDistribution[0] = new Dictionary <string, double> {
                { LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 }
            };

            // One-versus-rest
            Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[0], actualLabelDistribution, actualLabelDistribution));
            Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[1], actualLabelDistribution, actualLabelDistribution));
            Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[2], actualLabelDistribution, actualLabelDistribution));

            // One-versus-another
            Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[0], LabelSet[2], actualLabelDistribution, actualLabelDistribution));
            Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[1], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
            Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[2], LabelSet[1], actualLabelDistribution, actualLabelDistribution));

            // Positive and negative class labels are identical
            Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
        }
Exemplo n.º 4
0
 public void TestNormalizedDcgBest()
 {
     double[] gains = { 7, 5, 4, 4, 4, 1 };
     Assert.Equal(1, Metrics.Ndcg(gains, gains, Metrics.LinearDiscountFunc), Tolerance);
     Assert.Equal(1, Metrics.Ndcg(gains, gains, Metrics.LogarithmicDiscountFunc), Tolerance);
 }