public void TestMeanDcg()
{
Func <IEnumerable <double>, double> linearDcg = list => Metrics.Dcg(list, Metrics.LinearDiscountFunc);
double expectedDcg = (5.3 + 3.1 + 1.9) / 3.0;
Assert.Equal(
expectedDcg,
this.evaluator.ItemRecommendationMetric(this.dataset, BuildRecommendationDictionary(this.dataset, 1), linearDcg),
CompareEps);
expectedDcg += (1.1 + 4.1) / (3.0 * 2.0);
Assert.Equal(
expectedDcg,
this.evaluator.ItemRecommendationMetric(this.dataset, BuildRecommendationDictionary(this.dataset, 2), linearDcg),
CompareEps);
expectedDcg += 4.7 / (3.0 * 3.0);
Assert.Equal(
expectedDcg,
this.evaluator.ItemRecommendationMetric(this.dataset, BuildRecommendationDictionary(this.dataset, 3), linearDcg),
CompareEps);
expectedDcg += 3.6 / (3.0 * 4.0);
Assert.Equal(
expectedDcg,
this.evaluator.ItemRecommendationMetric(this.dataset, BuildRecommendationDictionary(this.dataset, 4), linearDcg),
CompareEps);
Assert.Equal(
expectedDcg,
this.evaluator.ItemRecommendationMetric(this.dataset, BuildRecommendationDictionary(this.dataset, 5), linearDcg),
CompareEps);
}
public void TestMeanNdcg()
{
Func <IEnumerable <double>, IEnumerable <double>, double> linearNdcg =
(list, bestList) => Metrics.Ndcg(list, bestList, Metrics.LinearDiscountFunc);
Assert.Equal(
((5.3 / 5.3) + (3.1 / 4.1) + (1.9 / 1.9)) / 3.0,
this.evaluator.ItemRecommendationMetric(this.dataset, BuildRecommendationDictionary(this.dataset, 1), linearNdcg),
CompareEps);
Assert.Equal(
(((5.3 + (1.1 / 2.0)) / (5.3 + (4.7 / 2.0))) + ((3.1 + (4.1 / 2.0)) / (4.1 + (3.1 / 2.0))) + (1.9 / 1.9)) / 3.0,
this.evaluator.ItemRecommendationMetric(this.dataset, BuildRecommendationDictionary(this.dataset, 2), linearNdcg),
CompareEps);
Assert.Equal(
(((5.3 + (1.1 / 2.0) + (4.7 / 3.0)) / (5.3 + (4.7 / 2.0) + (3.6 / 3.0))) + ((3.1 + (4.1 / 2.0)) / (4.1 + (3.1 / 2.0))) + (1.9 / 1.9)) / 3.0,
this.evaluator.ItemRecommendationMetric(this.dataset, BuildRecommendationDictionary(this.dataset, 3), linearNdcg),
CompareEps);
Assert.Equal(
(((5.3 + (1.1 / 2.0) + (4.7 / 3.0) + (3.6 / 4.0)) / (5.3 + (4.7 / 2.0) + (3.6 / 3.0) + (1.1 / 4.0))) + ((3.1 + (4.1 / 2.0)) / (4.1 + (3.1 / 2.0))) + (1.9 / 1.9)) / 3.0,
this.evaluator.ItemRecommendationMetric(this.dataset, BuildRecommendationDictionary(this.dataset, 4), linearNdcg),
CompareEps);
Assert.Equal(
(((5.3 + (1.1 / 2.0) + (4.7 / 3.0) + (3.6 / 4.0)) / (5.3 + (4.7 / 2.0) + (3.6 / 3.0) + (1.1 / 4.0))) + ((3.1 + (4.1 / 2.0)) / (4.1 + (3.1 / 2.0))) + (1.9 / 1.9)) / 3.0,
this.evaluator.ItemRecommendationMetric(this.dataset, BuildRecommendationDictionary(this.dataset, 5), linearNdcg),
CompareEps);
}
public void TestExpectedMeanAbsoluteError()
{
double expectedMae = this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, this.uncertainPredictions, Metrics.AbsoluteError);
const double TrueExpectedMae = (2.6 + 0.2 + 1.4 + 1.5 + 1.5 + 0.8 + 0.0) / 7.0;
Assert.Equal(TrueExpectedMae, expectedMae, CompareEps);
}
public void TestModelDomainMeanSquaredError()
{
double mse = this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, this.predictions, Metrics.SquaredError);
const double TrueMse = (9.0 + 0.0 + 4.0 + 9.0 + 9.0 + 1.0 + 0.0) / 7.0;
Assert.Equal(TrueMse, mse, CompareEps);
}
public void TestDataDomainMeanSquaredError()
{
double mse = this.evaluator.RatingPredictionMetric(this.dataset, this.predictions, Metrics.SquaredError);
const double TrueMse = (8.41 + 0.09 + 2.89 + 6.76 + 9.61 + 1.21 + 0.01) / 7.0;
Assert.Equal(TrueMse, mse, CompareEps);
}
public void TestExpectedMeanSquaredError()
{
double expectedMse = this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, this.uncertainPredictions, Metrics.SquaredError);
const double TrueExpectedMse = (8.6 + 0.2 + 2.6 + 4.5 + 4.5 + 0.8 + 0.0) / 7.0;
Assert.Equal(TrueExpectedMse, expectedMse, CompareEps);
}
public void TestDataDomainMeanAbsoluteError()
{
double mae = this.evaluator.RatingPredictionMetric(this.dataset, this.predictions, Metrics.AbsoluteError);
const double TrueMae = (2.9 + 0.3 + 1.7 + 2.6 + 3.1 + 1.1 + 0.1) / 7.0;
Assert.Equal(TrueMae, mae, CompareEps);
}
public void TestExpectedWeightedConfusion()
{
double expectedAbsoluteErrorAsWeightedConfusion = this.evaluator.ExpectedWeightedConfusion(
this.dataset,
this.uncertainPredictions,
RatingMatrix.AbsoluteErrorLossMatrix(this.starRatingInfo.MinStarRating, this.starRatingInfo.MaxStarRating));
Assert.Equal(
this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, this.uncertainPredictions, Metrics.AbsoluteError),
expectedAbsoluteErrorAsWeightedConfusion,
CompareEps);
double expectedSquaredErrorAsWeightedConfusion = this.evaluator.ExpectedWeightedConfusion(
this.dataset,
this.uncertainPredictions,
RatingMatrix.SquaredErrorLossMatrix(this.starRatingInfo.MinStarRating, this.starRatingInfo.MaxStarRating));
Assert.Equal(
this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, this.uncertainPredictions, Metrics.SquaredError),
expectedSquaredErrorAsWeightedConfusion,
CompareEps);
double expectedZeroOneErrorAsWeightedConfusion = this.evaluator.ExpectedWeightedConfusion(
this.dataset,
this.uncertainPredictions,
RatingMatrix.ZeroOneErrorLossMatrix(this.starRatingInfo.MinStarRating, this.starRatingInfo.MaxStarRating));
Assert.Equal(
this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, this.uncertainPredictions, Metrics.ZeroOneError),
expectedZeroOneErrorAsWeightedConfusion,
CompareEps);
}
public void TestModelDomainMeanAbsoluteError()
{
double mae = this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, this.predictions, Metrics.AbsoluteError);
const double TrueMae = (3.0 + 0.0 + 2.0 + 3.0 + 3.0 + 1.0 + 0.0) / 7.0;
Assert.Equal(TrueMae, mae, CompareEps);
}
public void TestItemFractions()
{
const int UserCount = 10;
const int ItemCount = 1000;
const double IgnoredItemFraction = 0.1;
const double ColdItemFraction = 0.2;
const double FractionTolerance = 1.0 / (UserCount * ItemCount);
var datasets = TestSplittingHelper(UserCount, ItemCount, 1.0, 0.5, 0.5, 0.0, ColdItemFraction, 0.0, IgnoredItemFraction, false);
// Ignored items
int ignoredItemCount =
datasets.Item1.Select(t => t.Item2) // All items
.Except(datasets.Item2.Select(t => t.Item2)) // Minus training items
.Except(datasets.Item3.Select(t => t.Item2)) // Minus test items
.Count();
Assert.Equal(IgnoredItemFraction, ignoredItemCount / (double)ItemCount, FractionTolerance);
// Cold items
int coldItemCount =
datasets.Item3.Select(t => t.Item2) // Test items
.Except(datasets.Item2.Select(t => t.Item2)) // Minus training items
.Count();
Assert.Equal(ColdItemFraction, coldItemCount / (double)ItemCount, FractionTolerance);
}
public void TestMeanNegativeLogProbability()
{
double negativeLogProb = this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, this.uncertainPredictions, Metrics.NegativeLogProbability);
double trueLogProb = (Math.Log(0.2) + Math.Log(0.8) + Math.Log(0.2) + Math.Log(0.5) + Math.Log(0.5) + Math.Log(0.2) + Math.Log(1.0)) / 7.0;
Assert.Equal(-trueLogProb, negativeLogProb, CompareEps);
}
public void TestProbabilityCalibrationError()
{
double error = this.evaluator.ProbabilityCalibrationError(this.dataset, this.uncertainPredictions, 5, 2);
const double TrueError = ((0.25 - (1.0 / 6.0)) + (1.0 - 0.75)) / 2.0;
Assert.Equal(TrueError, error);
}
public void TestAreaUnderRocCurve()
{
// Duplicate instance scores, duplicate positive instances
Assert.Equal(0.75, Metrics.AreaUnderRocCurve(new[] { 1, 1, 2 }, new Dictionary <int, double> {
{ 1, 0.5 }, { 2, 0.5 }, { 3, 0.5 }, { 4, 0 }
}));
Assert.Equal(0.5, Metrics.AreaUnderRocCurve(new[] { 1, 1, 2 }, new Dictionary <int, double> {
{ 1, 0.5 }, { 2, 0.5 }, { 3, 1 }, { 4, 0 }
}));
// No positive instance scores
Assert.Throws <ArgumentException>(() => Metrics.AreaUnderRocCurve(new int[] { }, new Dictionary <int, double> {
{ 1, 1 }
}));
// No negative instance scores
Assert.Throws <ArgumentException>(() => Metrics.AreaUnderRocCurve(new[] { 1 }, new Dictionary <int, double> {
{ 1, 1 }
}));
// No instance scores
Assert.Throws <ArgumentException>(() => Metrics.AreaUnderRocCurve(new[] { 1 }, new Dictionary <int, double>()));
// Null checks
Assert.Throws <ArgumentNullException>(() => Metrics.AreaUnderRocCurve(null, new Dictionary <int, double> {
{ 1, 1 }
}));
Assert.Throws <ArgumentNullException>(() => Metrics.AreaUnderRocCurve(new[] { 1 }, null));
}
/// <summary>
/// Verifies that two specified curves are equal.
/// </summary>
/// <param name="expected">The expected curve.</param>
/// <param name="actual">The actual curve.</param>
private static void AssertCurvesAreEqual(IList <Pair <double, double> > expected, IList <Pair <double, double> > actual)
{
Assert.Equal(expected.Count, actual.Count);
for (int point = 0; point < actual.Count; point++)
{
Assert.Equal(expected[point], actual[point]);
}
}
public void TestNormalizedManhattanDistance()
{
double[] values1 = { 3, 1, 2, 2 };
double[] values2 = { 1, 2, 1, 0 };
double distance = Metrics.NormalizedManhattanSimilarity(Vector.FromArray(values1), Vector.FromArray(values2));
const double TrueDistance = 0.4;
Assert.Equal(TrueDistance, distance, Tolerance);
}
public void TestNormalizedEuclideanDistance()
{
double[] values1 = { 3, 1, 2, 2 };
double[] values2 = { 1, 2, 1, 0 };
double distance = Metrics.NormalizedEuclideanSimilarity(Vector.FromArray(values1), Vector.FromArray(values2));
double trueDistance = 1.0 / (1.0 + Math.Sqrt(2.5));
Assert.Equal(trueDistance, distance, Tolerance);
}
public void TestPearsonCorrelation()
{
double[] values1 = { 3, 1, 2, 2 };
double[] values2 = { 1, 2, 1, 0 };
double correlation = Metrics.PearsonCorrelation(Vector.FromArray(values1), Vector.FromArray(values2));
const double TrueCorrelation = -0.5;
Assert.Equal(TrueCorrelation, correlation, Tolerance);
}
public void TestCosineSimilarity()
{
double[] values1 = { 3, 1, 2, 2 };
double[] values2 = { 1, 1, 2, -2 };
double sim = Metrics.CosineSimilarity(Vector.FromArray(values1), Vector.FromArray(values2));
double trueSim = 4 / Math.Sqrt(18) / Math.Sqrt(10);
Assert.Equal(trueSim, sim, Tolerance);
}
public void TestGap()
{
double[] relevances = { 3, 1, 2, 2 };
double gap = Metrics.GradedAveragePrecision(relevances);
const double TrueGap =
((3.0 / 1.0) + ((1.0 + 1.0) / 2.0) + ((2.0 + 1.0 + 2.0) / 3.0) + ((2.0 + 1.0 + 2.0 + 2.0) / 4.0)) / (3.0 + 1.0 + 2.0 + 2.0);
Assert.Equal(TrueGap, gap, Tolerance);
}
public void TestDiscretePointEstimator()
{
var distribution = new Discrete(1, 1, 4, 2);
Assert.Equal(2, distribution.GetMode());
Assert.Equal(2, PointEstimator.GetEstimate(distribution, Metrics.ZeroOneError)); // Mode
Assert.Equal(2, Convert.ToInt32(distribution.GetMean()));
Assert.Equal(2, PointEstimator.GetEstimate(distribution, Metrics.SquaredError)); // Mean
Assert.Equal(2, distribution.GetMedian());
Assert.Equal(2, PointEstimator.GetEstimate(distribution, Metrics.AbsoluteError)); // Median
Assert.Equal(2, PointEstimator.GetEstimate(distribution, this.LinearLossFunction())); // Median
Assert.Equal(1, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(3))); // 1st quartile
Assert.Equal(2, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 3))); // 3rd quartile
Assert.Equal(3, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 4))); // 4th quintile
distribution = new Discrete(0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1);
Assert.Equal(0, distribution.GetMode());
Assert.Equal(4.5, distribution.GetMean(), 1e-10);
int median = distribution.GetMedian();
Assert.True(median == 4 || median == 5);
Assert.Equal(3, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(3, 2))); // 2nd quintile
Assert.Equal(7, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 3))); // 3rd quartile
Assert.Equal(9, PointEstimator.GetEstimate(distribution, this.LinearLossFunction(1, 999))); // 999th permille
Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(null, Metrics.AbsoluteError));
Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(distribution, null));
// Test generic representation of distribution
var genericDistribution = new Dictionary <int, double> {
{ 0, 1 }, { 1, 1 }, { 2, 4 }, { 3, 2 }
};
Assert.Equal(2, genericDistribution.GetMode());
Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode
Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, Metrics.SquaredError)); // Mean
Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, Metrics.AbsoluteError)); // Median
Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction())); // Median
Assert.Equal(1, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(3))); // 1st quartile
Assert.Equal(2, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 3))); // 3rd quartile
Assert.Equal(3, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 4))); // 4th quintile
genericDistribution = new Dictionary <int, double> {
{ 0, 0.1 }, { 1, 0.1 }, { 2, 0.1 }, { 3, 0.1 }, { 4, 0.1 }, { 5, 0.1 }, { 6, 0.1 }, { 7, 0.1 }, { 8, 0.1 }, { 9, 0.1 }
};
Assert.Equal(0, genericDistribution.GetMode());
Assert.Equal(0, PointEstimator.GetEstimate(genericDistribution, Metrics.ZeroOneError)); // Mode
Assert.Equal(4, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction())); // Median
Assert.Equal(3, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(3, 2))); // 2nd quintile
Assert.Equal(7, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 3))); // 3rd quartile
Assert.Equal(9, PointEstimator.GetEstimate(genericDistribution, this.LinearLossFunction(1, 999))); // 999th permille
Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(null, Metrics.AbsoluteError));
Assert.Throws <ArgumentNullException>(() => PointEstimator.GetEstimate(genericDistribution, null));
}
public void ConfusionMatrixTest()
{
string expectedConfusionMatrixString = "Truth \\ Prediction ->" + Environment.NewLine +
" A B C" + Environment.NewLine +
" A 3 0 0" + Environment.NewLine +
" B 1 0 0" + Environment.NewLine +
" C 1 0 0" + Environment.NewLine;
var predictedLabels = new[] { LabelSet[0], LabelSet[0], LabelSet[0], LabelSet[0], LabelSet[0] };
var confusionMatrix = this.evaluator.ConfusionMatrix(this.groundTruth, predictedLabels);
// Verify ToString method
Assert.Equal(expectedConfusionMatrixString, confusionMatrix.ToString());
// Counts
Assert.Equal(3, confusionMatrix[LabelSet[0], LabelSet[0]]);
Assert.Equal(0, confusionMatrix[LabelSet[0], LabelSet[1]]);
Assert.Equal(0, confusionMatrix[LabelSet[0], LabelSet[2]]);
Assert.Equal(1, confusionMatrix[LabelSet[1], LabelSet[0]]);
Assert.Equal(0, confusionMatrix[LabelSet[1], LabelSet[1]]);
Assert.Equal(0, confusionMatrix[LabelSet[1], LabelSet[2]]);
Assert.Equal(1, confusionMatrix[LabelSet[2], LabelSet[0]]);
Assert.Equal(0, confusionMatrix[LabelSet[2], LabelSet[1]]);
Assert.Equal(0, confusionMatrix[LabelSet[2], LabelSet[2]]);
// Precision
Assert.Equal(0.6, confusionMatrix.Precision(LabelSet[0]), Tolerance);
Assert.Equal(double.NaN, confusionMatrix.Precision(LabelSet[1])); // undefined result
Assert.Equal(double.NaN, confusionMatrix.Precision(LabelSet[2])); // undefined result
Assert.Equal(0.6, confusionMatrix.MacroPrecision, Tolerance);
Assert.Equal(0.36, confusionMatrix.MicroPrecision, Tolerance);
// Recall
Assert.Equal(1, confusionMatrix.Recall(LabelSet[0]));
Assert.Equal(0, confusionMatrix.Recall(LabelSet[1]));
Assert.Equal(0, confusionMatrix.Recall(LabelSet[2]));
Assert.Equal(1 / 3.0, confusionMatrix.MacroRecall, Tolerance);
Assert.Equal(0.6, confusionMatrix.MicroRecall, Tolerance);
// Accuracy
Assert.Equal(1, confusionMatrix.Accuracy(LabelSet[0]));
Assert.Equal(0, confusionMatrix.Accuracy(LabelSet[1]));
Assert.Equal(0, confusionMatrix.Accuracy(LabelSet[2]));
Assert.Equal(1 / 3.0, confusionMatrix.MacroAccuracy, Tolerance);
Assert.Equal(0.6, confusionMatrix.MicroAccuracy, Tolerance);
// F1-measure
Assert.Equal(0.75, confusionMatrix.F1(LabelSet[0]), Tolerance);
Assert.Equal(double.NaN, confusionMatrix.F1(LabelSet[1])); // undefined result
Assert.Equal(double.NaN, confusionMatrix.F1(LabelSet[2])); // undefined result
Assert.Equal(0.75, confusionMatrix.MacroF1, Tolerance);
Assert.Equal(0.45, confusionMatrix.MicroF1, Tolerance);
}
public void TestDcg()
{
double[] gains = { 3, 1, 2, 2 };
double dcg = Metrics.Dcg(gains, Metrics.LinearDiscountFunc);
double trueDcg = (3.0 / 1.0) + (1.0 / 2.0) + (2.0 / 3.0) + (2.0 / 4.0);
Assert.Equal(trueDcg, dcg, Tolerance);
// DCG is defined even for empty sequences
Assert.Equal(0, Metrics.Dcg(new double[0]));
}
public void TestPerUserMetricAggregation()
{
double perUserMae = this.evaluator.ModelDomainRatingPredictionMetric(this.dataset, this.predictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst);
const double TruePerUserMae = (((3.0 + 0.0 + 2.0 + 3.0) / 4.0) + ((3.0 + 1.0) / 2.0) + (0.0 / 1.0)) / 3.0;
Assert.Equal(TruePerUserMae, perUserMae, CompareEps);
double perUserExpectedMae = this.evaluator.ModelDomainRatingPredictionMetricExpectation(this.dataset, this.uncertainPredictions, Metrics.AbsoluteError, RecommenderMetricAggregationMethod.PerUserFirst);
const double TruePerUserExpectedMae = (((2.6 + 0.2 + 1.4 + 1.5) / 4.0) + ((1.5 + 0.8) / 2.0) + (0.0 / 3.0)) / 3.0;
Assert.Equal(TruePerUserExpectedMae, perUserExpectedMae, CompareEps);
}
public void TestEmptyItemRecommendations()
{
// No predictions to evaluate
var testRecommendations = new Dictionary <string, IEnumerable <string> >();
Assert.Throws <ArgumentException>(() => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg));
Assert.Throws <ArgumentException>(() => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Ndcg));
// One user with empty recommendation list is presented
testRecommendations.Add("A", new List <string>());
Assert.Equal(0.0, this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Dcg)); // DCG makes sense
Assert.Throws <ArgumentException>(() => this.evaluator.ItemRecommendationMetric(this.dataset, testRecommendations, Metrics.Ndcg)); // NDCG doesn't
}
public void TestAbsoluteError()
{
Assert.Equal(0, Metrics.AbsoluteError(1, 1), Tolerance);
Assert.Equal(0, Metrics.AbsoluteError(0, 0), Tolerance);
Assert.Equal(2, Metrics.AbsoluteError(3, 1), Tolerance);
Assert.Equal(3, Metrics.AbsoluteError(-5, -2), Tolerance);
Assert.Equal(4, Metrics.AbsoluteError(3, -1), Tolerance);
Assert.Equal(0, Metrics.AbsoluteError(true, true), Tolerance);
Assert.Equal(1, Metrics.AbsoluteError(false, true), Tolerance);
Assert.Equal(0, Metrics.AbsoluteError(false, false), Tolerance);
Assert.Equal(1, Metrics.AbsoluteError(true, false), Tolerance);
}
public void TestSquaredError()
{
Assert.Equal(0, Metrics.SquaredError(1, 1), Tolerance);
Assert.Equal(0, Metrics.SquaredError(0, 0), Tolerance);
Assert.Equal(4, Metrics.SquaredError(3, 1), Tolerance);
Assert.Equal(9, Metrics.SquaredError(-5, -2), Tolerance);
Assert.Equal(16, Metrics.SquaredError(3, -1), Tolerance);
Assert.Equal(0, Metrics.SquaredError(true, true), Tolerance);
Assert.Equal(1, Metrics.SquaredError(false, true), Tolerance);
Assert.Equal(0, Metrics.SquaredError(false, false), Tolerance);
Assert.Equal(1, Metrics.SquaredError(true, false), Tolerance);
}
public void TestProbabilityCalibrationPlot()
{
const int Bins = 3;
double[] plot = this.evaluator.ProbabilityCalibrationPlot(this.dataset, this.uncertainPredictions, 4, Bins);
Assert.Equal(Bins, plot.Length);
Assert.False(double.IsNaN(plot[0]));
Assert.False(double.IsNaN(plot[1]));
Assert.True(double.IsNaN(plot[2]));
Assert.Equal(0, plot[0], CompareEps);
Assert.Equal(2.0 / 3.0, plot[1], CompareEps);
}
public void TestRelatedItemsSaveLoad()
{
var relatedItems =
new Dictionary <Item, IEnumerable <Item> >
{
{ new Item("1", null), new[] { new Item("1", null), new Item("aba", null) } },
{ new Item("adbda", null), new[] { new Item("2", null) } }
};
const string FileName = "related_items.txt";
RecommenderPersistenceUtils.SaveRelatedItems(FileName, relatedItems);
var loadedRelatedItems = RecommenderPersistenceUtils.LoadRelatedItems(FileName);
Assert.Equal(relatedItems, loadedRelatedItems, Comparers.Collection);
}
public void TestRelatedItemsDcgAt2()
{
var relatedItemPredictions = new Dictionary <string, IEnumerable <string> > {
{ "a", new[] { "c", "b" } }
};
double ndcg = this.evaluator.RelatedItemsMetric(
this.dataset, relatedItemPredictions, 1, Metrics.LinearNdcg, Metrics.CosineSimilarity, Math.Round);
double dcg = this.evaluator.RelatedItemsMetric(
this.dataset, relatedItemPredictions, 1, Metrics.LinearDcg, Metrics.CosineSimilarity, Math.Round);
double expectedDcg = (1.0 / Math.Sqrt(2.0)) + 0.5;
const double BestDcg = 1.0 + 0.5;
Assert.Equal(expectedDcg, dcg, CompareEps);
Assert.Equal(expectedDcg / BestDcg, ndcg, CompareEps);
}
public void TestRelatedUsersDcgAt2()
{
var relatedUserPredictions = new Dictionary <string, IEnumerable <string> > {
{ "A", new[] { "B", "D" } }
};
double ndcg = this.evaluator.RelatedUsersMetric(
this.dataset, relatedUserPredictions, 1, Metrics.LinearNdcg, Metrics.CosineSimilarity, Math.Round);
double dcg = this.evaluator.RelatedUsersMetric(
this.dataset, relatedUserPredictions, 1, Metrics.LinearDcg, Metrics.CosineSimilarity, Math.Round);
double expectedDcg = (19.0 / (5.0 * Math.Sqrt(26.0))) + 0.5;
double bestDcg = 1.0 + ((0.5 * 19.0) / (5.0 * Math.Sqrt(26.0)));
Assert.Equal(expectedDcg, dcg, CompareEps);
Assert.Equal(expectedDcg / bestDcg, ndcg, CompareEps);
}
