public void TestIncorrectFractionSums() { Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0.5, 0, 0.1, 0, false)); Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0.6, 0, 0, 0, false)); Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0, 0.5, 0, 0.6, false)); }
public void TestPrecisionRecallCurve() { // General var expectedCurve = new[] { new PrecisionRecall(1, 0), new PrecisionRecall(0, 0), new PrecisionRecall(0.5, 0.5), new PrecisionRecall(2 / (double)3, 1), new PrecisionRecall(0.5, 1) }; var computedCurve = Metrics.PrecisionRecallCurve(new[] { 1, 2 }, new Dictionary <int, double> { { 3, 1 }, { 1, 0.5 }, { 2, 0.25 }, { 4, 0 } }).ToArray(); Assert.Equal(expectedCurve.Length, computedCurve.Length); for (int i = 0; i < expectedCurve.Length; i++) { Assert.Equal(expectedCurve[i], computedCurve[i]); } // No instance scores expectedCurve = new[] { new PrecisionRecall(1, 0), }; computedCurve = Metrics.PrecisionRecallCurve(new[] { 1 }, new Dictionary <int, double>()).ToArray(); Assert.Equal(expectedCurve.Length, computedCurve.Length); for (int i = 0; i < expectedCurve.Length; i++) { Assert.Equal(expectedCurve[i], computedCurve[i]); } // No negative instance scores expectedCurve = new[] { new PrecisionRecall(1, 0), new PrecisionRecall(1, 1) }; computedCurve = Metrics.PrecisionRecallCurve(new[] { 1 }, new Dictionary <int, double> { { 1, 1 } }).ToArray(); Assert.Equal(expectedCurve.Length, computedCurve.Length); for (int i = 0; i < expectedCurve.Length; i++) { Assert.Equal(expectedCurve[i], computedCurve[i]); } // Duplicate positive instances computedCurve = Metrics.PrecisionRecallCurve(new[] { 1, 1 }, new Dictionary <int, double> { { 1, 1 } }).ToArray(); Assert.Equal(expectedCurve.Length, computedCurve.Length); for (int i = 0; i < expectedCurve.Length; i++) { Assert.Equal(expectedCurve[i], computedCurve[i]); } // No positive instance scores Assert.Throws <ArgumentException>(() => Metrics.PrecisionRecallCurve(new int[] { }, new Dictionary <int, double> { { 1, 1 } })); // Null checks Assert.Throws <ArgumentNullException>(() => Metrics.PrecisionRecallCurve(null, new Dictionary <int, double> { { 1, 1 } })); Assert.Throws <ArgumentNullException>(() => Metrics.PrecisionRecallCurve(new[] { 1 }, null)); }
public void AreaUnderRocCurveTest() { // AUC for perfect predictions // Per-label AUC Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[0], this.groundTruth, this.groundTruth)); Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[1], this.groundTruth, this.groundTruth)); Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[2], this.groundTruth, this.groundTruth)); // M-measure IDictionary <string, IDictionary <string, double> > computedAucMatrix; Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.groundTruth, out computedAucMatrix)); Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.groundTruth)); // Pairwise AUC (upper triangle) Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[2]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[2]]); // Pairwise AUC (diagnonal) foreach (string label in LabelSet) { Assert.Equal(double.NaN, computedAucMatrix[label][label]); // undefined result } // Pairwise AUC (lower triangle) Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[2]][LabelSet[0]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[2]][LabelSet[1]]); // AUC for imperfect predictions // Per-label AUC Assert.Equal(5 / 12.0, this.evaluator.AreaUnderRocCurve(LabelSet[0], this.groundTruth, this.predictions)); // matches ROC curve Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[1], this.groundTruth, this.predictions)); Assert.Equal(1 / 8.0, this.evaluator.AreaUnderRocCurve(LabelSet[2], this.groundTruth, this.predictions)); // M-measure Assert.Equal(5 / 9.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.predictions, out computedAucMatrix)); Assert.Equal(5 / 9.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.predictions)); // Pairwise AUC (upper triangle) Assert.Equal(5 / 6.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]); // matches ROC curve Assert.Equal(0.0, computedAucMatrix[LabelSet[0]][LabelSet[2]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[2]]); // Pairwise AUC (diagnonal) foreach (string label in LabelSet) { Assert.Equal(double.NaN, computedAucMatrix[label][label]); // undefined result } // Pairwise AUC (lower triangle) Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]); Assert.Equal(0.0, computedAucMatrix[LabelSet[2]][LabelSet[0]]); Assert.Equal(0.5, computedAucMatrix[LabelSet[2]][LabelSet[1]]); // Test code path for symmetric two-class case var binaryGroundTruth = new LabelDistribution[2]; binaryGroundTruth[0] = new Dictionary <string, double> { { LabelSet[0], 1 }, { LabelSet[1], 0 } }; binaryGroundTruth[1] = new Dictionary <string, double> { { LabelSet[0], 0 }, { LabelSet[1], 1 } }; var binaryPredictions = new LabelDistribution[2]; binaryPredictions[0] = new Dictionary <string, double> { { LabelSet[0], 0.9 }, { LabelSet[1], 0.1 } }; binaryPredictions[1] = new Dictionary <string, double> { { LabelSet[0], 0.8 }, { LabelSet[1], 0.2 } }; Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(binaryGroundTruth, binaryPredictions, out computedAucMatrix)); Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(binaryGroundTruth, binaryPredictions)); Assert.Equal(double.NaN, computedAucMatrix[LabelSet[0]][LabelSet[0]]); // undefined result Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]); Assert.Equal(double.NaN, computedAucMatrix[LabelSet[1]][LabelSet[1]]); // undefined result // No positive or negative class labels var actualLabelDistribution = new LabelDistribution[1]; actualLabelDistribution[0] = new Dictionary <string, double> { { LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 } }; // One-versus-rest Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[0], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[1], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[2], actualLabelDistribution, actualLabelDistribution)); // One-versus-another Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[0], LabelSet[2], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[1], LabelSet[0], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[2], LabelSet[1], actualLabelDistribution, actualLabelDistribution)); // Positive and negative class labels are identical Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[0], actualLabelDistribution, actualLabelDistribution)); }
public void TestNormalizedDcgBest() { double[] gains = { 7, 5, 4, 4, 4, 1 }; Assert.Equal(1, Metrics.Ndcg(gains, gains, Metrics.LinearDiscountFunc), Tolerance); Assert.Equal(1, Metrics.Ndcg(gains, gains, Metrics.LogarithmicDiscountFunc), Tolerance); }