public void TestDiscretePointEstimator()
{
var distribution = new Discrete(1, 1, 4, 2);
Assert.Equal(2, distribution.GetMode());
Assert.Equal(2, PointEstimator.GetEstimate(distribution, Metrics.ZeroOneError)); // Mode
Assert.Equal(2, Convert.ToInt32(distribution.GetMean()));
Assert.Equal(2, PointEstimator.GetEstimate(distribution, Metrics.SquaredError)); // Mean
Assert.Equal(2, distribution.GetMedian());
Assert.Equal(2, PointEstimator.GetEstimate(distribution, Metrics.AbsoluteError)); // Median
Assert.Equal(2, PointEstimator.GetEstimate(distribution, this.LinearLossFunction())); // Median
Assert.Equal(1, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(3))); // 1st quartile
Assert.Equal(2, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 3))); // 3rd quartile
Assert.Equal(3, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 4))); // 4th quintile
distribution = new Discrete(0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1);
Assert.Equal(0, distribution.GetMode());
Assert.Equal(5, Convert.ToInt32(distribution.GetMean()));
Assert.Equal(4, distribution.GetMedian());
Assert.Equal(3, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(3, 2))); // 2nd quintile
Assert.Equal(7, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 3))); // 3rd quartile
Assert.Equal(9, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 999))); // 999th permille
Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(null, Metrics.AbsoluteError));
Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(distribution, null));
// Test generic representation of distribution
var genericDistribution = new Dictionary <int, double> {
{ 0, 1 }, { 1, 1 }, { 2, 4 }, { 3, 2 }
};
Assert.Equal(2, genericDistribution.GetMode());
Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode
Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, Metrics.SquaredError)); // Mean
Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, Metrics.AbsoluteError)); // Median
Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction())); // Median
Assert.Equal(1, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(3))); // 1st quartile
Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 3))); // 3rd quartile
Assert.Equal(3, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 4))); // 4th quintile
genericDistribution = new Dictionary <int, double> {
{ 0, 0.1 }, { 1, 0.1 }, { 2, 0.1 }, { 3, 0.1 }, { 4, 0.1 }, { 5, 0.1 }, { 6, 0.1 }, { 7, 0.1 }, { 8, 0.1 }, { 9, 0.1 }
};
Assert.Equal(0, genericDistribution.GetMode());
Assert.Equal(0, PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode
Assert.Equal(4, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction())); // Median
Assert.Equal(3, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(3, 2))); // 2nd quintile
Assert.Equal(7, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 3))); // 3rd quartile
Assert.Equal(9, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 999))); // 999th permille
Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(null, Metrics.AbsoluteError));
Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(genericDistribution, null));
}
public void TestInvalidNdcgComputationCalls()
{
Assert.Throws <ArgumentNullException>(() => Metrics.LinearNdcg(null, new double[0])); // Null arguments 1
Assert.Throws <ArgumentNullException>(() => Metrics.LinearNdcg(new double[0], null)); // Null arguments 2
Assert.Throws <ArgumentNullException>(() => Metrics.LinearNdcg(null, null)); // Null arguments 3
Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new double[0], new double[0])); // Empty gain lists
Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new[] { 1.0 }, new[] { 0.0 })); // Denominator is zero 1
Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new[] { 0.0 }, new[] { 0.0 })); // Denominator is zero 2
Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new[] { 1.0, 1.0 }, new[] { 2.0, -4.0 })); // Denominator is zero 3
Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new[] { 1.0, 1.0 }, new[] { 1.0, -4.0 })); // NDCG is negative
Assert.Throws <ArgumentException>(() => Metrics.LinearNdcg(new[] { 1.0, 2.0 }, new[] { 1.0, 1.0 })); // NDCG is greater than 1
}
public void TestEmptyItemRecommendations()
{
// No predictions to evaluate
var testRecommendations = new Dictionary <string, IEnumerable <string> >();
Assert.Throws <ArgumentException>(() => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg));
Assert.Throws <ArgumentException>(() => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Ndcg));
// One user with empty recommendation list is presented
testRecommendations.Add("A", new List <string>());
Assert.Equal(0.0, this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg)); // DCG makes sense
Assert.Throws <ArgumentException>(() => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Ndcg)); // NDCG doesn't
}
public void TestBernoulliPointEstimator()
{
var distribution = new Bernoulli(0.25);
Assert.False(distribution.GetMode());
Assert.False(PointEstimator.GetEstimate(distribution, Metrics.ZeroOneError)); // Mode
Assert.False(distribution.GetMean() >= 0.5);
Assert.False(PointEstimator.GetEstimate(distribution, Metrics.SquaredError)); // Mean
Assert.False(PointEstimator.GetEstimate(distribution, Metrics.AbsoluteError)); // Median = Mode
distribution = new Bernoulli(0.5);
Assert.True(distribution.GetMode());
Assert.True(PointEstimator.GetEstimate(distribution, Metrics.ZeroOneError)); // Mode
Assert.True(distribution.GetMean() >= 0.5);
Assert.True(PointEstimator.GetEstimate(distribution, Metrics.SquaredError)); // Mean
Assert.True(PointEstimator.GetEstimate(distribution, Metrics.AbsoluteError)); // Median = Mode
distribution = new Bernoulli(0.6);
Assert.True(distribution.GetMode());
Assert.True(PointEstimator.GetEstimate(distribution, Metrics.ZeroOneError)); // Mode
Assert.True(distribution.GetMean() >= 0.5);
Assert.True(PointEstimator.GetEstimate(distribution, Metrics.SquaredError)); // Mean
Assert.True(PointEstimator.GetEstimate(distribution, Metrics.AbsoluteError)); // Median = Mode
Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(distribution, null));
// Test generic representation of distribution
var genericDistribution = new Dictionary <bool, double> {
{ true, 0.25 }, { false, 0.75 }
}; // Order matters!
Assert.False(PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode
Assert.False(PointEstimator.GetEstimate(genericDistribution, Metrics.SquaredError)); // Mean
Assert.False(PointEstimator.GetEstimate(genericDistribution, Metrics.AbsoluteError)); // Median = Mode
genericDistribution = new Dictionary <bool, double> {
{ true, 0.5 }, { false, 0.5 }
};
Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode
Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.SquaredError)); // Mean
Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.AbsoluteError)); // Median = Mode
genericDistribution = new Dictionary <bool, double> {
{ true, 0.6 }, { false, 0.4 }
};
Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode
Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.SquaredError)); // Mean
Assert.True(PointEstimator.GetEstimate(genericDistribution, Metrics.AbsoluteError)); // Median = Mode
Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(distribution, null));
}
public void TestQueryItemInRelatedItemList()
{
var testPredictionsGood = new Dictionary <string, IEnumerable <string> > {
{ "a", new[] { "b" } }
};
this.evaluator.RelatedItemsMetric(this.dataset, testPredictionsGood, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity); // Must not throw
var testPredictionsBad = new Dictionary <string, IEnumerable <string> > {
{ "a", new[] { "a" } }
};
Assert.Throws <ArgumentException>(
() => this.evaluator.RelatedItemsMetric(this.dataset, testPredictionsBad, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity)); // Must throw, A is being related to itself
}
public void TestReturnRelatedItemMoreThanOnce()
{
var relatedItemsForA = new List <string>();
var testPredictions = new Dictionary <string, IEnumerable <string> > {
{ "a", relatedItemsForA }
};
this.evaluator.RelatedItemsMetric(this.dataset, testPredictions, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity); // Must not throw
relatedItemsForA.Add("b");
this.evaluator.RelatedItemsMetric(this.dataset, testPredictions, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity); // Must not throw
relatedItemsForA.Add("c");
this.evaluator.RelatedItemsMetric(this.dataset, testPredictions, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity); // Must not throw
relatedItemsForA.Add("b");
Assert.Throws <ArgumentException>(() => this.evaluator.RelatedItemsMetric(this.dataset, testPredictions, 1, Metrics.Dcg, Metrics.NormalizedEuclideanSimilarity)); // Must throw, b is presented twice
}
public void TestRecommendItemMoreThanOnce()
{
var recommendationsForA = new List <string>();
var testRecommendations = new Dictionary <string, IEnumerable <string> > {
{ "A", recommendationsForA }
};
this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg); // Must not throw
recommendationsForA.Add("a");
this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg); // Must not throw
recommendationsForA.Add("b");
this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg); // Must not throw
recommendationsForA.Add("a");
Assert.Throws <ArgumentException>(
() => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg)); // Must throw, 'a' is recommended twice
}
public void TestEmptyRelatedItemPredictions()
{
// No predictions to evaluate
var testRelatedItems = new Dictionary <string, IEnumerable <string> >();
Assert.Throws <ArgumentException>(
() => this.evaluator.RelatedItemsMetric(this.dataset, testRelatedItems, 1, Metrics.Dcg, Metrics.NormalizedManhattanSimilarity));
Assert.Throws <ArgumentException>(
() => this.evaluator.RelatedItemsMetric(this.dataset, testRelatedItems, 1, Metrics.Ndcg, Metrics.NormalizedManhattanSimilarity));
// One item with empty list of related items is presented
testRelatedItems.Add("a", new List <string>());
Assert.Equal(
0.0, this.evaluator.RelatedItemsMetric(this.dataset, testRelatedItems, 1, Metrics.Dcg, Metrics.NormalizedManhattanSimilarity)); // DCG makes sense
Assert.Throws <ArgumentException>(
() => this.evaluator.RelatedItemsMetric(this.dataset, testRelatedItems, 1, Metrics.Ndcg, Metrics.NormalizedManhattanSimilarity)); // NDCG doesn't
}
public void TestNegativeLogProbability()
{
var bernoulliDistribution = new Bernoulli(0.25);
Assert.Equal(Math.Log(4) - Math.Log(3), Metrics.NegativeLogProbability(false, bernoulliDistribution), Tolerance);
Assert.Equal(Math.Log(4), Metrics.NegativeLogProbability(true, bernoulliDistribution), Tolerance);
var discreteDistribution = new Discrete(1.0, 1.0, 4.0, 2.0);
Assert.Equal(Math.Log(8), Metrics.NegativeLogProbability(0, discreteDistribution), Tolerance);
Assert.Equal(Math.Log(8), Metrics.NegativeLogProbability(1, discreteDistribution), Tolerance);
Assert.Equal(Math.Log(2), Metrics.NegativeLogProbability(2, discreteDistribution), Tolerance);
Assert.Equal(Math.Log(4), Metrics.NegativeLogProbability(3, discreteDistribution), Tolerance);
var labelDistribution = new Dictionary <string, double> {
{ "A", 0.125 }, { "B", 0.125 }, { "C", 0.5 }, { "D", 0.25 }
};
Assert.Equal(Math.Log(8), Metrics.NegativeLogProbability("A", labelDistribution), Tolerance);
Assert.Equal(Math.Log(8), Metrics.NegativeLogProbability("B", labelDistribution), Tolerance);
Assert.Equal(Math.Log(2), Metrics.NegativeLogProbability("C", labelDistribution), Tolerance);
Assert.Equal(Math.Log(4), Metrics.NegativeLogProbability("D", labelDistribution), Tolerance);
Assert.Throws <ArgumentException>(() => Metrics.NegativeLogProbability("E", labelDistribution));
Assert.Throws <ArgumentNullException>(() => Metrics.NegativeLogProbability("A", null));
Assert.Throws <ArgumentNullException>(() => Metrics.NegativeLogProbability(null, labelDistribution));
var labelDictionary = new Dictionary <string, double> {
{ "A", 0.125 }, { "B", 0.125 }, { "C", 0.5 }, { "D", 0.26 }
};
Assert.Throws <ArgumentException>(() => Metrics.NegativeLogProbability("D", labelDictionary));
var negativelabelDictionary = new Dictionary <string, double> {
{ "A", 0.5 }, { "B", 0.5 }, { "C", 0.5 }, { "D", -0.5 }
};
Assert.Throws <ArgumentException>(() => Metrics.NegativeLogProbability("D", negativelabelDictionary));
var shortLabelDictionary = new Dictionary <string, double> {
{ "A", 1 }
};
Assert.Throws <ArgumentException>(() => Metrics.NegativeLogProbability("A", shortLabelDictionary));
}
public void TestReceiverOperatingCharacteristicCurve()
{
// Duplicate instance scores, duplicate positive instances
var expectedCurve = new[]
{
Pair.Create <double, double>(0, 0),
Pair.Create <double, double>(0.5, 1),
Pair.Create <double, double>(1, 1)
};
var computedCurve = Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1, 1, 2 }, new Dictionary <int, double> {
{ 1, 0.5 }, { 2, 0.5 }, { 3, 0.5 }, { 4, 0 }
}).ToArray();
foreach (var tuple in computedCurve)
{
Console.WriteLine(tuple);
}
Assert.Equal(expectedCurve.Length, computedCurve.Length);
for (int i = 0; i < expectedCurve.Length; i++)
{
Assert.Equal(expectedCurve[i], computedCurve[i]);
}
// No positive instance scores
Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new int[] { }, new Dictionary <int, double> {
{ 1, 1 }
}));
// No negative instance scores
Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, new Dictionary <int, double> {
{ 1, 1 }
}));
// No instance scores
Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, new Dictionary <int, double>()));
// Null checks
Assert.Throws <ArgumentNullException>(() => Metrics.ReceiverOperatingCharacteristicCurve(null, new Dictionary <int, double> {
{ 1, 1 }
}));
Assert.Throws <ArgumentNullException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, null));
}
public void TestZeroOneError()
{
Assert.Equal(0, Metrics.ZeroOneError(1, 1), Tolerance);
Assert.Equal(0, Metrics.ZeroOneError(0, 0), Tolerance);
Assert.Equal(1, Metrics.ZeroOneError(3, 1), Tolerance);
Assert.Equal(1, Metrics.ZeroOneError(-5, -2), Tolerance);
Assert.Equal(1, Metrics.ZeroOneError(3, -1), Tolerance);
Assert.Equal(0, Metrics.ZeroOneError("A", "A"), Tolerance);
Assert.Equal(1, Metrics.ZeroOneError("A", "B"), Tolerance);
Assert.Equal(0, Metrics.ZeroOneError(true, true), Tolerance);
Assert.Equal(1, Metrics.ZeroOneError(false, true), Tolerance);
Assert.Equal(0, Metrics.ZeroOneError(false, false), Tolerance);
Assert.Equal(1, Metrics.ZeroOneError(true, false), Tolerance);
Assert.Throws <ArgumentNullException>(() => Metrics.ZeroOneError("A", null));
Assert.Throws <ArgumentNullException>(() => Metrics.ZeroOneError(null, "B"));
}
public void PerformanceMetricEvaluationTest()
{
// Perfect predictions
double negativeLogProbability = this.evaluator.Evaluate(this.groundTruth, this.groundTruth, Metrics.NegativeLogProbability);
Assert.Equal(0.0, negativeLogProbability);
// Imperfect predictions
negativeLogProbability = this.evaluator.Evaluate(this.groundTruth, this.predictions, Metrics.NegativeLogProbability);
Assert.Equal(double.PositiveInfinity, negativeLogProbability);
var uncertainPredictions = new LabelDistribution[5];
uncertainPredictions[0] = new Dictionary <string, double> {
{ LabelSet[0], 0.5 }, { LabelSet[1], 0.25 }, { LabelSet[2], 0.25 }
};
uncertainPredictions[1] = new Dictionary <string, double> {
{ LabelSet[0], 0 }, { LabelSet[1], 1 }, { LabelSet[2], 0 }
};
uncertainPredictions[2] = new Dictionary <string, double> {
{ LabelSet[0], 0.25 }, { LabelSet[1], 0.25 }, { LabelSet[2], 0.5 }
};
uncertainPredictions[3] = new Dictionary <string, double> {
{ LabelSet[0], 1 / 6.0 }, { LabelSet[1], 2 / 3.0 }, { LabelSet[2], 1 / 6.0 }
};
uncertainPredictions[4] = new Dictionary <string, double> {
{ LabelSet[0], 1 / 8.0 }, { LabelSet[1], 1 / 8.0 }, { LabelSet[2], 3 / 4.0 }
};
negativeLogProbability = this.evaluator.Evaluate(this.groundTruth, uncertainPredictions, Metrics.NegativeLogProbability);
Assert.Equal(5.2574953720277815, negativeLogProbability, Tolerance);
// Insufficient number of predictions
var insufficientPredictions = new LabelDistribution[1];
insufficientPredictions[0] = new Dictionary <string, double> {
{ LabelSet[0], 0.5 }, { LabelSet[1], 0.25 }, { LabelSet[2], 0.25 }
};
Assert.Throws <ArgumentException>(() => this.evaluator.Evaluate(this.groundTruth, insufficientPredictions, Metrics.NegativeLogProbability));
}
public void TestReceiverOperatingCharacteristicCurve()
{
// Duplicate instance scores, duplicate positive instances
var expectedCurve = new[]
{
new FalseAndTruePositiveRate(0, 0),
new FalseAndTruePositiveRate(0.5, 1),
new FalseAndTruePositiveRate(1, 1)
};
var computedCurve = Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1, 1, 2 }, new Dictionary <int, double> {
{ 1, 0.5 }, { 2, 0.5 }, { 3, 0.5 }, { 4, 0 }
}).ToArray();
foreach (var tuple in computedCurve)
{
Console.WriteLine(tuple);
}
Xunit.Assert.Equal(expectedCurve, computedCurve);
// No positive instance scores
Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new int[] { }, new Dictionary <int, double> {
{ 1, 1 }
}));
// No negative instance scores
Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, new Dictionary <int, double> {
{ 1, 1 }
}));
// No instance scores
Assert.Throws <ArgumentException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, new Dictionary <int, double>()));
// Null checks
Assert.Throws <ArgumentNullException>(() => Metrics.ReceiverOperatingCharacteristicCurve(null, new Dictionary <int, double> {
{ 1, 1 }
}));
Assert.Throws <ArgumentNullException>(() => Metrics.ReceiverOperatingCharacteristicCurve(new[] { 1 }, null));
}
public void TestEmptyRatingPredictions()
{
// No predictions to evaluate
var testPredictions = new Dictionary <string, IDictionary <string, int> >();
var testUncertainPredictions = new Dictionary <string, IDictionary <string, RatingDistribution> >();
Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError));
Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst));
Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError));
Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst));
// There is one user presented but still no predictions to evaluate
testPredictions.Add("A", new Dictionary <string, int>());
testUncertainPredictions.Add("A", new Dictionary <string, RatingDistribution>());
Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError));
Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst));
Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError));
Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst));
// Now we add one more user with predictions
testPredictions.Add("B", new Dictionary <string, int> {
{ "a", 4 }
});
testUncertainPredictions.Add(
"B", new Dictionary <string, RatingDistribution> {
{ "a", new SortedDictionary <int, double> {
{ 0, 0.0 }, { 1, 0.0 }, { 2, 0.0 }, { 3, 0.0 }, { 4, 1.0 }, { 5, 0.0 }
} }
});
// If there are some predictions, default aggregation makes sense
Assert.Equal(0.0, this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError));
Assert.Equal(0.0, this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError));
// Per-user aggregation still doesn't make sense
Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, testPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst));
Assert.Throws <ArgumentException>(() => this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, testUncertainPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst));
}
public void RocCurveTest()
{
// Curve for perfect predictions
var expected = new[] { new FalseAndTruePositiveRate(0.0, 0.0), new FalseAndTruePositiveRate(0.0, 1.0), new FalseAndTruePositiveRate(1.0, 1.0) };
var actual = this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], this.groundTruth, this.groundTruth).ToArray();
Xunit.Assert.Equal(expected, actual);
// Curve for imperfect predictions (one-versus-rest)
expected = new[] { new FalseAndTruePositiveRate(0.0, 0.0), new FalseAndTruePositiveRate(0.5, 0.0), new FalseAndTruePositiveRate(0.5, 1 / 3.0), new FalseAndTruePositiveRate(0.5, 2 / 3.0), new FalseAndTruePositiveRate(1.0, 1.0) };
actual = this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], this.groundTruth, this.predictions).ToArray();
Xunit.Assert.Equal(expected, actual); // matches below AUC = 5/12
// Curve for imperfect predictions (one-versus-another)
expected = new[] { new FalseAndTruePositiveRate(0.0, 0.0), new FalseAndTruePositiveRate(0.0, 1 / 3.0), new FalseAndTruePositiveRate(0.0, 2 / 3.0), new FalseAndTruePositiveRate(1.0, 1.0) };
actual = this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[1], this.groundTruth, this.predictions).ToArray();
Xunit.Assert.Equal(expected, actual); // matches below AUC = 5/6
// No positive or negative class labels
var actualLabelDistribution = new LabelDistribution[1];
actualLabelDistribution[0] = new Dictionary <string, double> {
{ LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 }
};
// One-versus-rest
Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[1], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[2], actualLabelDistribution, actualLabelDistribution));
// One-versus-another
Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[2], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[1], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[2], LabelSet[1], actualLabelDistribution, actualLabelDistribution));
// Positive and negative class labels are identical
Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
}
public void PrecisionRecallCurveTest()
{
// Curve for perfect predictions
var expected = new[] { new PrecisionRecall(1.0, 0.0), new PrecisionRecall(1.0, 1 / 3.0), new PrecisionRecall(1.0, 2 / 3.0), new PrecisionRecall(1.0, 1.0), new PrecisionRecall(0.75, 1.0), new PrecisionRecall(0.6, 1.0) };
var actual = this.evaluator.PrecisionRecallCurve(LabelSet[0], this.groundTruth, this.groundTruth).ToArray();
Xunit.Assert.Equal(expected, actual);
// Curve for imperfect predictions (one-versus-rest)
expected = new[] { new PrecisionRecall(1.0, 0.0), new PrecisionRecall(0.0, 0.0), new PrecisionRecall(0.5, 1 / 3.0), new PrecisionRecall(2 / 3.0, 2 / 3.0), new PrecisionRecall(0.75, 1.0), new PrecisionRecall(0.6, 1.0) };
actual = this.evaluator.PrecisionRecallCurve(LabelSet[0], this.groundTruth, this.predictions).ToArray();
Xunit.Assert.Equal(expected, actual);
// Curve for imperfect predictions (one-versus-another)
expected = new[] { new PrecisionRecall(1.0, 0.0), new PrecisionRecall(1.0, 1.0), new PrecisionRecall(0.5, 1.0) };
actual = this.evaluator.PrecisionRecallCurve(LabelSet[1], LabelSet[2], this.groundTruth, this.predictions).ToArray();
Xunit.Assert.Equal(expected, actual);
// No positive class labels
var actualLabelDistribution = new LabelDistribution[1];
actualLabelDistribution[0] = new Dictionary <string, double> {
{ LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 }
};
// One-versus-rest
Assert.Throws <ArgumentException>(() => this.evaluator.PrecisionRecallCurve(LabelSet[1], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.PrecisionRecallCurve(LabelSet[2], actualLabelDistribution, actualLabelDistribution));
// One-versus-another
Assert.Throws <ArgumentException>(() => this.evaluator.PrecisionRecallCurve(LabelSet[1], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.PrecisionRecallCurve(LabelSet[2], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
// Positive and negative class labels are identical
Assert.Throws <ArgumentException>(() => this.evaluator.PrecisionRecallCurve(LabelSet[0], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
}
public void AreaUnderRocCurveTest()
{
// AUC for perfect predictions
// Per-label AUC
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[0], this.groundTruth, this.groundTruth));
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[1], this.groundTruth, this.groundTruth));
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[2], this.groundTruth, this.groundTruth));
// M-measure
IDictionary <string, IDictionary <string, double> > computedAucMatrix;
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.groundTruth, out computedAucMatrix));
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.groundTruth));
// Pairwise AUC (upper triangle)
Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[2]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[2]]);
// Pairwise AUC (diagnonal)
foreach (string label in LabelSet)
{
Assert.Equal(double.NaN, computedAucMatrix[label][label]); // undefined result
}
// Pairwise AUC (lower triangle)
Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[2]][LabelSet[0]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[2]][LabelSet[1]]);
// AUC for imperfect predictions
// Per-label AUC
Assert.Equal(5 / 12.0, this.evaluator.AreaUnderRocCurve(LabelSet[0], this.groundTruth, this.predictions)); // matches ROC curve
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(LabelSet[1], this.groundTruth, this.predictions));
Assert.Equal(1 / 8.0, this.evaluator.AreaUnderRocCurve(LabelSet[2], this.groundTruth, this.predictions));
// M-measure
Assert.Equal(5 / 9.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.predictions, out computedAucMatrix));
Assert.Equal(5 / 9.0, this.evaluator.AreaUnderRocCurve(this.groundTruth, this.predictions));
// Pairwise AUC (upper triangle)
Assert.Equal(5 / 6.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]); // matches ROC curve
Assert.Equal(0.0, computedAucMatrix[LabelSet[0]][LabelSet[2]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[2]]);
// Pairwise AUC (diagnonal)
foreach (string label in LabelSet)
{
Assert.Equal(double.NaN, computedAucMatrix[label][label]); // undefined result
}
// Pairwise AUC (lower triangle)
Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]);
Assert.Equal(0.0, computedAucMatrix[LabelSet[2]][LabelSet[0]]);
Assert.Equal(0.5, computedAucMatrix[LabelSet[2]][LabelSet[1]]);
// Test code path for symmetric two-class case
var binaryGroundTruth = new LabelDistribution[2];
binaryGroundTruth[0] = new Dictionary <string, double> {
{ LabelSet[0], 1 }, { LabelSet[1], 0 }
};
binaryGroundTruth[1] = new Dictionary <string, double> {
{ LabelSet[0], 0 }, { LabelSet[1], 1 }
};
var binaryPredictions = new LabelDistribution[2];
binaryPredictions[0] = new Dictionary <string, double> {
{ LabelSet[0], 0.9 }, { LabelSet[1], 0.1 }
};
binaryPredictions[1] = new Dictionary <string, double> {
{ LabelSet[0], 0.8 }, { LabelSet[1], 0.2 }
};
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(binaryGroundTruth, binaryPredictions, out computedAucMatrix));
Assert.Equal(1.0, this.evaluator.AreaUnderRocCurve(binaryGroundTruth, binaryPredictions));
Assert.Equal(double.NaN, computedAucMatrix[LabelSet[0]][LabelSet[0]]); // undefined result
Assert.Equal(1.0, computedAucMatrix[LabelSet[0]][LabelSet[1]]);
Assert.Equal(1.0, computedAucMatrix[LabelSet[1]][LabelSet[0]]);
Assert.Equal(double.NaN, computedAucMatrix[LabelSet[1]][LabelSet[1]]); // undefined result
// No positive or negative class labels
var actualLabelDistribution = new LabelDistribution[1];
actualLabelDistribution[0] = new Dictionary <string, double> {
{ LabelSet[0], 1 }, { LabelSet[1], 0 }, { LabelSet[2], 0 }
};
// One-versus-rest
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[0], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[1], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[2], actualLabelDistribution, actualLabelDistribution));
// One-versus-another
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[0], LabelSet[2], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[1], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
Assert.Throws <ArgumentException>(() => this.evaluator.AreaUnderRocCurve(LabelSet[2], LabelSet[1], actualLabelDistribution, actualLabelDistribution));
// Positive and negative class labels are identical
Assert.Throws <ArgumentException>(() => this.evaluator.ReceiverOperatingCharacteristicCurve(LabelSet[0], LabelSet[0], actualLabelDistribution, actualLabelDistribution));
}
public void TestPrecisionRecallCurve()
{
// General
var expectedCurve = new[]
{
new PrecisionRecall(1, 0),
new PrecisionRecall(0, 0),
new PrecisionRecall(0.5, 0.5),
new PrecisionRecall(2 / (double)3, 1),
new PrecisionRecall(0.5, 1)
};
var computedCurve = Metrics.PrecisionRecallCurve(new[] { 1, 2 }, new Dictionary <int, double> {
{ 3, 1 }, { 1, 0.5 }, { 2, 0.25 }, { 4, 0 }
}).ToArray();
Assert.Equal(expectedCurve.Length, computedCurve.Length);
for (int i = 0; i < expectedCurve.Length; i++)
{
Assert.Equal(expectedCurve[i], computedCurve[i]);
}
// No instance scores
expectedCurve = new[] { new PrecisionRecall(1, 0), };
computedCurve = Metrics.PrecisionRecallCurve(new[] { 1 }, new Dictionary <int, double>()).ToArray();
Assert.Equal(expectedCurve.Length, computedCurve.Length);
for (int i = 0; i < expectedCurve.Length; i++)
{
Assert.Equal(expectedCurve[i], computedCurve[i]);
}
// No negative instance scores
expectedCurve = new[] { new PrecisionRecall(1, 0), new PrecisionRecall(1, 1) };
computedCurve = Metrics.PrecisionRecallCurve(new[] { 1 }, new Dictionary <int, double> {
{ 1, 1 }
}).ToArray();
Assert.Equal(expectedCurve.Length, computedCurve.Length);
for (int i = 0; i < expectedCurve.Length; i++)
{
Assert.Equal(expectedCurve[i], computedCurve[i]);
}
// Duplicate positive instances
computedCurve = Metrics.PrecisionRecallCurve(new[] { 1, 1 }, new Dictionary <int, double> {
{ 1, 1 }
}).ToArray();
Assert.Equal(expectedCurve.Length, computedCurve.Length);
for (int i = 0; i < expectedCurve.Length; i++)
{
Assert.Equal(expectedCurve[i], computedCurve[i]);
}
// No positive instance scores
Assert.Throws <ArgumentException>(() => Metrics.PrecisionRecallCurve(new int[] { }, new Dictionary <int, double> {
{ 1, 1 }
}));
// Null checks
Assert.Throws <ArgumentNullException>(() => Metrics.PrecisionRecallCurve(null, new Dictionary <int, double> {
{ 1, 1 }
}));
Assert.Throws <ArgumentNullException>(() => Metrics.PrecisionRecallCurve(new[] { 1 }, null));
}
public void TestIncorrectFractionSums()
{
Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0.5, 0, 0.1, 0, false));
Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0.6, 0, 0, 0, false));
Assert.Throws <ArgumentException>(() => TestSplittingHelper(1, 1, 1, 0.5, 0.25, 0, 0.5, 0, 0.6, false));
}
