public void TestDiscretePointEstimator() { var distribution = new Discrete(1, 1, 4, 2); Assert.Equal(2, distribution.GetMode()); Assert.Equal(2, PointEstimator.GetEstimate(distribution, Metrics.ZeroOneError)); // Mode Assert.Equal(2, Convert.ToInt32(distribution.GetMean())); Assert.Equal(2, PointEstimator.GetEstimate(distribution, Metrics.SquaredError)); // Mean Assert.Equal(2, distribution.GetMedian()); Assert.Equal(2, PointEstimator.GetEstimate(distribution, Metrics.AbsoluteError)); // Median Assert.Equal(2, PointEstimator.GetEstimate(distribution, this.LinearLossFunction())); // Median Assert.Equal(1, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(3))); // 1st quartile Assert.Equal(2, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 3))); // 3rd quartile Assert.Equal(3, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 4))); // 4th quintile distribution = new Discrete(0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1); Assert.Equal(0, distribution.GetMode()); Assert.Equal(5, Convert.ToInt32(distribution.GetMean())); Assert.Equal(4, distribution.GetMedian()); Assert.Equal(3, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(3, 2))); // 2nd quintile Assert.Equal(7, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 3))); // 3rd quartile Assert.Equal(9, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 999))); // 999th permille Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(null, Metrics.AbsoluteError)); Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(distribution, null)); // Test generic representation of distribution var genericDistribution = new Dictionary <int, double> { { 0, 1 }, { 1, 1 }, { 2, 4 }, { 3, 2 } }; Assert.Equal(2, genericDistribution.GetMode()); Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, Metrics.SquaredError)); // Mean Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, Metrics.AbsoluteError)); // Median Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction())); // Median Assert.Equal(1, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(3))); // 1st quartile Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 3))); // 3rd quartile Assert.Equal(3, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 4))); // 4th quintile genericDistribution = new Dictionary <int, double> { { 0, 0.1 }, { 1, 0.1 }, { 2, 0.1 }, { 3, 0.1 }, { 4, 0.1 }, { 5, 0.1 }, { 6, 0.1 }, { 7, 0.1 }, { 8, 0.1 }, { 9, 0.1 } }; Assert.Equal(0, genericDistribution.GetMode()); Assert.Equal(0, PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode Assert.Equal(4, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction())); // Median Assert.Equal(3, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(3, 2))); // 2nd quintile Assert.Equal(7, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 3))); // 3rd quartile Assert.Equal(9, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 999))); // 999th permille Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(null, Metrics.AbsoluteError)); Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(genericDistribution, null)); }
public void TestInvalidNdcgComputationCalls() { Assert.Throws <ArgumentNullException>(() => Metrics.LinearNdcg(null, new double[0])); // Null arguments 1 Assert.Throws <ArgumentNullException>(() => Metrics.LinearNdcg(new double[0], null)); // Null arguments 2 Assert.Throws <ArgumentNullException>(() => Metrics.LinearNdcg(null, null)); // Null arguments 3 Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new double[0], new double[0])); // Empty gain lists Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new[] { 1.0 }, new[] { 0.0 })); // Denominator is zero 1 Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new[] { 0.0 }, new[] { 0.0 })); // Denominator is zero 2 Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new[] { 1.0, 1.0 }, new[] { 2.0, -4.0 })); // Denominator is zero 3 Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new[] { 1.0, 1.0 }, new[] { 1.0, -4.0 })); // NDCG is negative Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new[] { 1.0, 2.0 }, new[] { 1.0, 1.0 })); // NDCG is greater than 1 }
public void TestEmptyItemRecommendations() { // No predictions to evaluate var testRecommendations = new Dictionary <string, IEnumerable <string> >(); Assert.Throws <ArgumentException>(() => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg)); Assert.Throws <ArgumentException>(() => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Ndcg)); // One user with empty recommendation list is presented testRecommendations.Add("A", new List <string>()); Assert.Equal(0.0, this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg)); // DCG makes sense Assert.Throws <ArgumentException>(() => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Ndcg)); // NDCG doesn't }
public void TestBernoulliPointEstimator() { var distribution = new Bernoulli(0.25); Assert.False(distribution.GetMode()); Assert.False(PointEstimator.GetEstimate(distribution, Metrics.ZeroOneError)); // Mode Assert.False(distribution.GetMean() >= 0.5); Assert.False(PointEstimator.GetEstimate(distribution, Metrics.SquaredError)); // Mean Assert.False(PointEstimator.GetEstimate(distribution, Metrics.AbsoluteError)); // Median = Mode distribution = new Bernoulli(0.5); Assert.True(distribution.GetMode()); Assert.True(PointEstimator.GetEstimate(distribution, Metrics.ZeroOneError)); // Mode Assert.True(distribution.GetMean() >= 0.5); Assert.True(PointEstimator.GetEstimate(distribution, Metrics.SquaredError)); // Mean Assert.True(PointEstimator.GetEstimate(distribution, Metrics.AbsoluteError)); // Median = Mode distribution = new Bernoulli(0.6); Assert.True(distribution.GetMode()); Assert.True(PointEstimator.GetEstimate(distribution, Metrics.ZeroOneError)); // Mode Assert.True(distribution.GetMean() >= 0.5); Assert.True(PointEstimator.GetEstimate(distribution, Metrics.SquaredError)); // Mean Assert.True(PointEstimator.GetEstimate(distribution, Metrics.AbsoluteError)); // Median = Mode Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(distribution, null)); // Test generic representation of distribution var genericDistribution = new Dictionary <bool, double> { { true, 0.25 }, { false, 0.75 } }; // Order matters! Assert.False(PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode Assert.False(PointEstimator.GetEstimate(genericDistribution, Metrics.SquaredError)); // Mean Assert.False(PointEstimator.GetEstimate(genericDistribution, Metrics.AbsoluteError)); // Median = Mode genericDistribution = new Dictionary <bool, double> { { true, 0.5 }, { false, 0.5 } }; Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.SquaredError)); // Mean Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.AbsoluteError)); // Median = Mode genericDistribution = new Dictionary <bool, double> { { true, 0.6 }, { false, 0.4 } }; Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.SquaredError)); // Mean Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.AbsoluteError)); // Median = Mode Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(distribution, null)); }
public void TestQueryItemInRelatedItemList() { var testPredictionsGood = new Dictionary <string, IEnumerable <string> > { { "a", new[] { "b" } } }; this.evaluator.RelatedItemsMetric(this.dataset, testPredictionsGood, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity); // Must not throw var testPredictionsBad = new Dictionary <string, IEnumerable <string> > { { "a", new[] { "a" } } }; Assert.Throws <ArgumentException>( () => this.evaluator.RelatedItemsMetric(this.dataset, testPredictionsBad, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity)); // Must throw, A is being related to itself }
public void TestReturnRelatedItemMoreThanOnce() { var relatedItemsForA = new List <string>(); var testPredictions = new Dictionary <string, IEnumerable <string> > { { "a", relatedItemsForA } }; this.evaluator.RelatedItemsMetric(this.dataset, testPredictions, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity); // Must not throw relatedItemsForA.Add("b"); this.evaluator.RelatedItemsMetric(this.dataset, testPredictions, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity); // Must not throw relatedItemsForA.Add("c"); this.evaluator.RelatedItemsMetric(this.dataset, testPredictions, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity); // Must not throw relatedItemsForA.Add("b"); Assert.Throws <ArgumentException>(() => this.evaluator.RelatedItemsMetric(this.dataset, testPredictions, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity)); // Must throw, b is presented twice }
public void TestRecommendItemMoreThanOnce() { var recommendationsForA = new List <string>(); var testRecommendations = new Dictionary <string, IEnumerable <string> > { { "A", recommendationsForA } }; this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg); // Must not throw recommendationsForA.Add("a"); this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg); // Must not throw recommendationsForA.Add("b"); this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg); // Must not throw recommendationsForA.Add("a"); Assert.Throws <ArgumentException>( () => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg)); // Must throw, 'a' is recommended twice }
public void TestEmptyRelatedItemPredictions() { // No predictions to evaluate var testRelatedItems = new Dictionary <string, IEnumerable <string> >(); Assert.Throws <ArgumentException>( () => this.evaluator.RelatedItemsMetric(this.dataset, testRelatedItems, 1, Metrics.Dcg, Metrics.NormalizedManhattanSimilarity)); Assert.Throws <ArgumentException>( () => this.evaluator.RelatedItemsMetric(this.dataset, testRelatedItems, 1, Metrics.Ndcg, Metrics.NormalizedManhattanSimilarity)); // One item with empty list of related items is presented testRelatedItems.Add("a", new List <string>()); Assert.Equal( 0.0, this.evaluator.RelatedItemsMetric(this.dataset, testRelatedItems, 1, Metrics.Dcg, Metrics.NormalizedManhattanSimilarity)); // DCG makes sense Assert.Throws <ArgumentException>( () => this.evaluator.RelatedItemsMetric(this.dataset, testRelatedItems, 1, Metrics.Ndcg, Metrics.NormalizedManhattanSimilarity)); // NDCG doesn't }
public void TestNegativeLogProbability() { var bernoulliDistribution = new Bernoulli(0.25); Assert.Equal(Math.Log(4) - Math.Log(3), Metrics.NegativeLogProbability(false, bernoulliDistribution), Tolerance); Assert.Equal(Math.Log(4), Metrics.NegativeLogProbability(true, bernoulliDistribution), Tolerance); var discreteDistribution = new Discrete(1.0, 1.0, 4.0, 2.0); Assert.Equal(Math.Log(8), Metrics.NegativeLogProbability(0, discreteDistribution), Tolerance); Assert.Equal(Math.Log(8), Metrics.NegativeLogProbability(1, discreteDistribution), Tolerance); Assert.Equal(Math.Log(2), Metrics.NegativeLogProbability(2, discreteDistribution), Tolerance); Assert.Equal(Math.Log(4), Metrics.NegativeLogProbability(3, discreteDistribution), Tolerance); var labelDistribution = new Dictionary <string, double> { { "A", 0.125 }, { "B", 0.125 }, { "C", 0.5 }, { "D", 0.25 } }; Assert.Equal(Math.Log(8), Metrics.NegativeLogProbability("A", labelDistribution), Tolerance); Assert.Equal(Math.Log(8), Metrics.NegativeLogProbability("B", labelDistribution), Tolerance); Assert.Equal(Math.Log(2), Metrics.NegativeLogProbability("C", labelDistribution), Tolerance); Assert.Equal(Math.Log(4), Metrics.NegativeLogProbability("D", labelDistribution), Tolerance); Assert.Throws <ArgumentException>(() => Metrics.NegativeLogProbability("E", labelDistribution)); Assert.Throws <ArgumentNullException>(() => Metrics.NegativeLogProbability("A", null)); Assert.Throws <ArgumentNullException>(() => Metrics.NegativeLogProbability(null, labelDistribution)); var labelDictionary = new Dictionary <string, double> { { "A", 0.125 }, { "B", 0.125 }, { "C", 0.5 }, { "D", 0.26 } }; Assert.Throws <ArgumentException>(() => Metrics.NegativeLogProbability("D", labelDictionary)); var negativelabelDictionary = new Dictionary <string, double> { { "A", 0.5 }, { "B", 0.5 }, { "C", 0.5 }, { "D", -0.5 } }; Assert.Throws <ArgumentException>(() => Metrics.NegativeLogProbability("D", negativelabelDictionary)); var shortLabelDictionary = new Dictionary <string, double> { { "A", 1 } }; Assert.Throws <ArgumentException>(() => Metrics.NegativeLogProbability("A", shortLabelDictionary)); }
public void TestReceiverOperatingCharacteristicCurve() { // Duplicate instance scores, duplicate positive instances var expectedCurve = new[] { Pair.Create <double, double>(0, 0), Pair.Create <double, double>(0.5, 1), Pair.Create <double, double>(1, 1) }; var computedCurve = Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1, 1, 2 }, new Dictionary <int, double> { { 1, 0.5 }, { 2, 0.5 }, { 3, 0.5 }, { 4, 0 } }).ToArray(); foreach (var tuple in computedCurve) { Console.WriteLine(tuple); } Assert.Equal(expectedCurve.Length, computedCurve.Length); for (int i = 0; i < expectedCurve.Length; i++) { Assert.Equal(expectedCurve[i], computedCurve[i]); } // No positive instance scores Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new int[] { }, new Dictionary <int, double> { { 1, 1 } })); // No negative instance scores Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, new Dictionary <int, double> { { 1, 1 } })); // No instance scores Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, new Dictionary <int, double>())); // Null checks Assert.Throws <ArgumentNullException>(() => Metrics.ReceiverOperatingCharacteristicCurve(null, new Dictionary <int, double> { { 1, 1 } })); Assert.Throws <ArgumentNullException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, null)); }
public void TestZeroOneError() { Assert.Equal(0, Metrics.ZeroOneError(1, 1), Tolerance); Assert.Equal(0, Metrics.ZeroOneError(0, 0), Tolerance); Assert.Equal(1, Metrics.ZeroOneError(3, 1), Tolerance); Assert.Equal(1, Metrics.ZeroOneError(-5, -2), Tolerance); Assert.Equal(1, Metrics.ZeroOneError(3, -1), Tolerance); Assert.Equal(0, Metrics.ZeroOneError("A", "A"), Tolerance); Assert.Equal(1, Metrics.ZeroOneError("A", "B"), Tolerance); Assert.Equal(0, Metrics.ZeroOneError(true, true), Tolerance); Assert.Equal(1, Metrics.ZeroOneError(false, true), Tolerance); Assert.Equal(0, Metrics.ZeroOneError(false, false), Tolerance); Assert.Equal(1, Metrics.ZeroOneError(true, false), Tolerance); Assert.Throws <ArgumentNullException>(() => Metrics.ZeroOneError("A", null)); Assert.Throws <ArgumentNullException>(() => Metrics.ZeroOneError(null, "B")); }
public void PerformanceMetricEvaluationTest() { // Perfect predictions double negativeLogProbability = this.evaluator.Evaluate(this.groundTruth, this.groundTruth, Metrics.NegativeLogProbability); Assert.Equal(0.0, negativeLogProbability); // Imperfect predictions negativeLogProbability = this.evaluator.Evaluate(this.groundTruth, this.predictions, Metrics.NegativeLogProbability); Assert.Equal(double.PositiveInfinity, negativeLogProbability); var uncertainPredictions = new LabelDistribution[5]; uncertainPredictions[0] = new Dictionary <string, double> { { LabelSet[0], 0.5 }, { LabelSet[1], 0.25 }, { LabelSet[2], 0.25 } }; uncertainPredictions[1] = new Dictionary <string, double> { { LabelSet[0], 0 }, { LabelSet[1], 1 }, { LabelSet[2], 0 } }; uncertainPredictions[2] = new Dictionary <string, double> { { LabelSet[0], 0.25 }, { LabelSet[1], 0.25 }, { LabelSet[2], 0.5 } }; uncertainPredictions[3] = new Dictionary <string, double> { { LabelSet[0], 1 / 6.0 }, { LabelSet[1], 2 / 3.0 }, { LabelSet[2], 1 / 6.0 } }; uncertainPredictions[4] = new Dictionary <string, double> { { LabelSet[0], 1 / 8.0 }, { LabelSet[1], 1 / 8.0 }, { LabelSet[2], 3 / 4.0 } }; negativeLogProbability = this.evaluator.Evaluate(this.groundTruth, uncertainPredictions, Metrics.NegativeLogProbability); Assert.Equal(5.2574953720277815, negativeLogProbability, Tolerance); // Insufficient number of predictions var insufficientPredictions = new LabelDistribution[1]; insufficientPredictions[0] = new Dictionary <string, double> { { LabelSet[0], 0.5 }, { LabelSet[1], 0.25 }, { LabelSet[2], 0.25 } }; Assert.Throws <ArgumentException>(() => this.evaluator.Evaluate(this.groundTruth, insufficientPredictions, Metrics.NegativeLogProbability)); }
public void TestReceiverOperatingCharacteristicCurve() { // Duplicate instance scores, duplicate positive instances var expectedCurve = new[] { new FalseAndTruePositiveRate(0, 0), new FalseAndTruePositiveRate(0.5, 1), new FalseAndTruePositiveRate(1, 1) }; var computedCurve = Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1, 1, 2 }, new Dictionary <int, double> { { 1, 0.5 }, { 2, 0.5 }, { 3, 0.5 }, { 4, 0 } }).ToArray(); foreach (var tuple in computedCurve) { Console.WriteLine(tuple); } Xunit.Assert.Equal(expectedCurve, computedCurve); // No positive instance scores Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new int[] { }, new Dictionary <int, double> { { 1, 1 } })); // No negative instance scores Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, new Dictionary <int, double> { { 1, 1 } })); // No instance scores Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, new Dictionary <int, double>())); // Null checks Assert.Throws <ArgumentNullException>(() => Metrics.ReceiverOperatingCharacteristicCurve(null, new Dictionary <int, double> { { 1, 1 } })); Assert.Throws <ArgumentNullException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, null)); }
public void TestEmptyRatingPredictions() { // No predictions to evaluate var testPredictions = new Dictionary <string, IDictionary <string, int> >(); var testUncertainPredictions = new Dictionary <string, IDictionary <string, RatingDistribution> >(); Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError)); Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst)); Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError)); Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst)); // There is one user presented but still no predictions to evaluate testPredictions.Add("A", new Dictionary <string, int>()); testUncertainPredictions.Add("A", new Dictionary <string, RatingDistribution>()); Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError)); Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst)); Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError)); Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst)); // Now we add one more user with predictions testPredictions.Add("B", new Dictionary <string, int> { { "a", 4 } }); testUncertainPredictions.Add( "B", new Dictionary <string, RatingDistribution> { { "a", new SortedDictionary <int, double> { { 0, 0.0 }, { 1, 0.0 }, { 2, 0.0 }, { 3, 0.0 }, { 4, 1.0 }, { 5, 0.0 } } } }); // If there are some predictions, default aggregation makes sense Assert.Equal(0.0, this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError)); Assert.Equal(0.0, this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError)); // Per-user aggregation still doesn't make sense Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst)); Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst)); }
public void RocCurveTest() { // Curve for perfect predictions var expected = new[] { new FalseAndTruePositiveRate(0.0, 0.0), new FalseAndTruePositiveRate(0.0, 1.0), new FalseAndTruePositiveRate(1.0, 1.0) }; var actual = this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], this.groundTruth, this.groundTruth).ToArray(); Xunit.Assert.Equal(expected, actual); // Curve for imperfect predictions (one-versus-rest) expected = new[] { new FalseAndTruePositiveRate(0.0, 0.0), new FalseAndTruePositiveRate(0.5, 0.0), new FalseAndTruePositiveRate(0.5, 1 / 3.0), new FalseAndTruePositiveRate(0.5, 2 / 3.0), new FalseAndTruePositiveRate(1.0, 1.0) }; actual = this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], this.groundTruth, this.predictions).ToArray(); Xunit.Assert.Equal(expected, actual); // matches below AUC = 5/12 // Curve for imperfect predictions (one-versus-another) expected = new[] { new FalseAndTruePositiveRate(0.0, 0.0), new FalseAndTruePositiveRate(0.0, 1 / 3.0), new FalseAndTruePositiveRate(0.0, 2 / 3.0), new FalseAndTruePositiveRate(1.0, 1.0) }; actual = this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[1], this.groundTruth, this.predictions).ToArray(); Xunit.Assert.Equal(expected, actual); // matches below AUC = 5/6 // No positive or negative class labels var actualLabelDistribution = new LabelDistribution[1]; actualLabelDistribution[0] = new Dictionary <string, double> { { LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 } }; // One-versus-rest Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[1], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[2], actualLabelDistribution, actualLabelDistribution)); // One-versus-another Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[2], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[1], LabelSet[0], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[2], LabelSet[1], actualLabelDistribution, actualLabelDistribution)); // Positive and negative class labels are identical Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[0], actualLabelDistribution, actualLabelDistribution)); }
public void PrecisionRecallCurveTest() { // Curve for perfect predictions var expected = new[] { new PrecisionRecall(1.0, 0.0), new PrecisionRecall(1.0, 1 / 3.0), new PrecisionRecall(1.0, 2 / 3.0), new PrecisionRecall(1.0, 1.0), new PrecisionRecall(0.75, 1.0), new PrecisionRecall(0.6, 1.0) }; var actual = this.evaluator.PrecisionRecallCurve(LabelSet[0], this.groundTruth, this.groundTruth).ToArray(); Xunit.Assert.Equal(expected, actual); // Curve for imperfect predictions (one-versus-rest) expected = new[] { new PrecisionRecall(1.0, 0.0), new PrecisionRecall(0.0, 0.0), new PrecisionRecall(0.5, 1 / 3.0), new PrecisionRecall(2 / 3.0, 2 / 3.0), new PrecisionRecall(0.75, 1.0), new PrecisionRecall(0.6, 1.0) }; actual = this.evaluator.PrecisionRecallCurve(LabelSet[0], this.groundTruth, this.predictions).ToArray(); Xunit.Assert.Equal(expected, actual); // Curve for imperfect predictions (one-versus-another) expected = new[] { new PrecisionRecall(1.0, 0.0), new PrecisionRecall(1.0, 1.0), new PrecisionRecall(0.5, 1.0) }; actual = this.evaluator.PrecisionRecallCurve(LabelSet[1], LabelSet[2], this.groundTruth, this.predictions).ToArray(); Xunit.Assert.Equal(expected, actual); // No positive class labels var actualLabelDistribution = new LabelDistribution[1]; actualLabelDistribution[0] = new Dictionary <string, double> { { LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 } }; // One-versus-rest Assert.Throws <ArgumentException>(() => this.evaluator.PrecisionRecallCurve(LabelSet[1], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.PrecisionRecallCurve(LabelSet[2], actualLabelDistribution, actualLabelDistribution)); // One-versus-another Assert.Throws <ArgumentException>(() => this.evaluator.PrecisionRecallCurve(LabelSet[1], LabelSet[0], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.PrecisionRecallCurve(LabelSet[2], LabelSet[0], actualLabelDistribution, actualLabelDistribution)); // Positive and negative class labels are identical Assert.Throws <ArgumentException>(() => this.evaluator.PrecisionRecallCurve(LabelSet[0], LabelSet[0], actualLabelDistribution, actualLabelDistribution)); }
public void AreaUnderRocCurveTest() { // AUC for perfect predictions // Per-label AUC Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[0], this.groundTruth, this.groundTruth)); Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[1], this.groundTruth, this.groundTruth)); Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[2], this.groundTruth, this.groundTruth)); // M-measure IDictionary <string, IDictionary <string, double> > computedAucMatrix; Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.groundTruth, out computedAucMatrix)); Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.groundTruth)); // Pairwise AUC (upper triangle) Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[2]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[2]]); // Pairwise AUC (diagnonal) foreach (string label in LabelSet) { Assert.Equal(double.NaN, computedAucMatrix[label][label]); // undefined result } // Pairwise AUC (lower triangle) Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[2]][LabelSet[0]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[2]][LabelSet[1]]); // AUC for imperfect predictions // Per-label AUC Assert.Equal(5 / 12.0, this.evaluator.AreaUnderRocCurve(LabelSet[0], this.groundTruth, this.predictions)); // matches ROC curve Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[1], this.groundTruth, this.predictions)); Assert.Equal(1 / 8.0, this.evaluator.AreaUnderRocCurve(LabelSet[2], this.groundTruth, this.predictions)); // M-measure Assert.Equal(5 / 9.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.predictions, out computedAucMatrix)); Assert.Equal(5 / 9.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.predictions)); // Pairwise AUC (upper triangle) Assert.Equal(5 / 6.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]); // matches ROC curve Assert.Equal(0.0, computedAucMatrix[LabelSet[0]][LabelSet[2]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[2]]); // Pairwise AUC (diagnonal) foreach (string label in LabelSet) { Assert.Equal(double.NaN, computedAucMatrix[label][label]); // undefined result } // Pairwise AUC (lower triangle) Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]); Assert.Equal(0.0, computedAucMatrix[LabelSet[2]][LabelSet[0]]); Assert.Equal(0.5, computedAucMatrix[LabelSet[2]][LabelSet[1]]); // Test code path for symmetric two-class case var binaryGroundTruth = new LabelDistribution[2]; binaryGroundTruth[0] = new Dictionary <string, double> { { LabelSet[0], 1 }, { LabelSet[1], 0 } }; binaryGroundTruth[1] = new Dictionary <string, double> { { LabelSet[0], 0 }, { LabelSet[1], 1 } }; var binaryPredictions = new LabelDistribution[2]; binaryPredictions[0] = new Dictionary <string, double> { { LabelSet[0], 0.9 }, { LabelSet[1], 0.1 } }; binaryPredictions[1] = new Dictionary <string, double> { { LabelSet[0], 0.8 }, { LabelSet[1], 0.2 } }; Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(binaryGroundTruth, binaryPredictions, out computedAucMatrix)); Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(binaryGroundTruth, binaryPredictions)); Assert.Equal(double.NaN, computedAucMatrix[LabelSet[0]][LabelSet[0]]); // undefined result Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]); Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]); Assert.Equal(double.NaN, computedAucMatrix[LabelSet[1]][LabelSet[1]]); // undefined result // No positive or negative class labels var actualLabelDistribution = new LabelDistribution[1]; actualLabelDistribution[0] = new Dictionary <string, double> { { LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 } }; // One-versus-rest Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[0], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[1], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[2], actualLabelDistribution, actualLabelDistribution)); // One-versus-another Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[0], LabelSet[2], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[1], LabelSet[0], actualLabelDistribution, actualLabelDistribution)); Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[2], LabelSet[1], actualLabelDistribution, actualLabelDistribution)); // Positive and negative class labels are identical Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[0], actualLabelDistribution, actualLabelDistribution)); }
public void TestPrecisionRecallCurve() { // General var expectedCurve = new[] { new PrecisionRecall(1, 0), new PrecisionRecall(0, 0), new PrecisionRecall(0.5, 0.5), new PrecisionRecall(2 / (double)3, 1), new PrecisionRecall(0.5, 1) }; var computedCurve = Metrics.PrecisionRecallCurve(new[] { 1, 2 }, new Dictionary <int, double> { { 3, 1 }, { 1, 0.5 }, { 2, 0.25 }, { 4, 0 } }).ToArray(); Assert.Equal(expectedCurve.Length, computedCurve.Length); for (int i = 0; i < expectedCurve.Length; i++) { Assert.Equal(expectedCurve[i], computedCurve[i]); } // No instance scores expectedCurve = new[] { new PrecisionRecall(1, 0), }; computedCurve = Metrics.PrecisionRecallCurve(new[] { 1 }, new Dictionary <int, double>()).ToArray(); Assert.Equal(expectedCurve.Length, computedCurve.Length); for (int i = 0; i < expectedCurve.Length; i++) { Assert.Equal(expectedCurve[i], computedCurve[i]); } // No negative instance scores expectedCurve = new[] { new PrecisionRecall(1, 0), new PrecisionRecall(1, 1) }; computedCurve = Metrics.PrecisionRecallCurve(new[] { 1 }, new Dictionary <int, double> { { 1, 1 } }).ToArray(); Assert.Equal(expectedCurve.Length, computedCurve.Length); for (int i = 0; i < expectedCurve.Length; i++) { Assert.Equal(expectedCurve[i], computedCurve[i]); } // Duplicate positive instances computedCurve = Metrics.PrecisionRecallCurve(new[] { 1, 1 }, new Dictionary <int, double> { { 1, 1 } }).ToArray(); Assert.Equal(expectedCurve.Length, computedCurve.Length); for (int i = 0; i < expectedCurve.Length; i++) { Assert.Equal(expectedCurve[i], computedCurve[i]); } // No positive instance scores Assert.Throws <ArgumentException>(() => Metrics.PrecisionRecallCurve(new int[] { }, new Dictionary <int, double> { { 1, 1 } })); // Null checks Assert.Throws <ArgumentNullException>(() => Metrics.PrecisionRecallCurve(null, new Dictionary <int, double> { { 1, 1 } })); Assert.Throws <ArgumentNullException>(() => Metrics.PrecisionRecallCurve(new[] { 1 }, null)); }
public void TestIncorrectFractionSums() { Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0.5, 0, 0.1, 0, false)); Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0.6, 0, 0, 0, false)); Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0, 0.5, 0, 0.6, false)); }