/// <summary> /// Computes the average of a given data domain rating prediction metric by iterating firstly per-item and then per-user. /// </summary> /// <param name="instanceSource">The source of the instances providing the ground truth.</param> /// <param name="predictions">A sparse users-by-items matrix of predicted rating distributions.</param> /// <param name="metric">The data domain rating prediction metric to average.</param> /// <param name="aggregationMethod">A method specifying how metrics are aggregated over the whole dataset.</param> /// <returns>The computed average of the given rating prediction metric.</returns> public double RatingPredictionMetric( TInstanceSource instanceSource, IDictionary <TUser, IDictionary <TItem, TPredictedRatingDist> > predictions, Func <TGroundTruthRating, TPredictedRatingDist, double> metric, RecommenderMetricAggregationMethod aggregationMethod = RecommenderMetricAggregationMethod.Default) { return(this.RatingPredictionMetric <TPredictedRatingDist>(instanceSource, predictions, metric, aggregationMethod)); }
/// <summary> /// Computes the expected component-wise product of a confusion matrix and a loss matrix. /// </summary> /// <param name="instanceSource">The instance source providing the ground truth (used to compute the expected confusion matrix).</param> /// <param name="predictions"> /// A sparse users-by-items matrix of predicted rating distributions (used to compute the expected confusion matrix). /// </param> /// <param name="lossMatrix">The loss matrix.</param> /// <param name="aggregationMethod"> /// A method specifying how metrics are aggregated over all instances (used to compute the expected confusion matrix).</param> /// <returns>The computed expected weighted confusion.</returns> public double ExpectedWeightedConfusion( TInstanceSource instanceSource, IDictionary <TUser, IDictionary <TItem, Discrete> > predictions, RatingMatrix lossMatrix, RecommenderMetricAggregationMethod aggregationMethod = RecommenderMetricAggregationMethod.Default) { RatingMatrix confusionMatrix = this.ExpectedConfusionMatrix(instanceSource, predictions, aggregationMethod); return(RatingMatrix.ComponentwiseProduct(confusionMatrix, lossMatrix)); }
/// <summary> /// Computes the average of a given rating prediction metric using ground truth in model domain by iterating over /// <paramref name="predictions"/> and using the aggregation method given in <paramref name="aggregationMethod"/>. /// </summary> /// <param name="instanceSource">The instance source providing the ground truth.</param> /// <param name="predictions">A sparse users-by-items matrix of predicted rating distributions.</param> /// <param name="metric">The rating prediction metric using ground truth in model domain.</param> /// <param name="aggregationMethod">A method specifying how metrics are aggregated over all instances.</param> /// <returns>The computed average of the given rating prediction metric.</returns> public double ModelDomainRatingPredictionMetric( TInstanceSource instanceSource, IDictionary <TUser, IDictionary <TItem, Discrete> > predictions, Func <int, Discrete, double> metric, RecommenderMetricAggregationMethod aggregationMethod = RecommenderMetricAggregationMethod.Default) { IStarRatingInfo <TGroundTruthRating> starRatingInfo = this.mapping.GetRatingInfo(instanceSource); Func <TGroundTruthRating, Discrete, double> metricWrapper = (g, up) => metric(starRatingInfo.ToStarRating(g), up); return(this.RatingPredictionMetric(instanceSource, predictions, metricWrapper, aggregationMethod)); }
/// <summary> /// Computes the average of a given rating prediction metric using ground truth in model domain by iterating over /// <paramref name="predictions"/> and using the aggregation method given in <paramref name="aggregationMethod"/>. /// </summary> /// <param name="instanceSource">The instance source providing the ground truth.</param> /// <param name="predictions">A sparse users-by-items matrix of predicted rating distributions.</param> /// <param name="metric">The rating prediction metric using ground truth in model domain.</param> /// <param name="aggregationMethod">A method specifying how metrics are aggregated over all instances.</param> /// <returns>The computed average of the given rating prediction metric.</returns> public double ModelDomainRatingPredictionMetricExpectation( TInstanceSource instanceSource, IDictionary <TUser, IDictionary <TItem, Discrete> > predictions, Func <int, int, double> metric, RecommenderMetricAggregationMethod aggregationMethod = RecommenderMetricAggregationMethod.Default) { return(this.ModelDomainRatingPredictionMetric( instanceSource, predictions, (r, ur) => RatingMetricExpectation(r, ur, metric), aggregationMethod)); }
/// <summary> /// Computes the average of a given rating prediction metric by iterating over /// <paramref name="predictions"/> and using the aggregation method given in <paramref name="aggregationMethod"/>. /// </summary> /// <typeparam name="TPrediction"> /// The type of predictions /// (used to support both certain and uncertain predictions). /// </typeparam> /// <param name="instanceSource">The source of the instances providing the ground truth.</param> /// <param name="predictions">A sparse users-by-items matrix of predicted rating distributions.</param> /// <param name="metric">The rating prediction metric to average.</param> /// <param name="aggregationMethod">A method specifying how metrics are aggregated over the whole dataset.</param> /// <returns>The computed average of the given rating prediction metric.</returns> private double RatingPredictionMetric <TPrediction>( TInstanceSource instanceSource, IDictionary <TUser, IDictionary <TItem, TPrediction> > predictions, Func <TGroundTruthRating, TPrediction, double> metric, RecommenderMetricAggregationMethod aggregationMethod) { double totalSum = 0; int totalCount = 0; foreach (var userWithPredictionList in predictions) { double perUserSum = 0; int perUserCount = 0; foreach (var itemPrediction in userWithPredictionList.Value) { TPrediction prediction = itemPrediction.Value; TGroundTruthRating groundTruth = this.mapping.GetRating(instanceSource, userWithPredictionList.Key, itemPrediction.Key); double metricValue = metric(groundTruth, prediction); perUserSum += metricValue; ++perUserCount; } switch (aggregationMethod) { case RecommenderMetricAggregationMethod.PerUserFirst: if (perUserCount == 0) { throw new ArgumentException("One of the users in the given predictions has no associated predictions."); } totalSum += perUserSum / perUserCount; ++totalCount; break; case RecommenderMetricAggregationMethod.Default: totalSum += perUserSum; totalCount += perUserCount; break; default: Debug.Fail("Metric aggregation method is not supported."); break; } } if (totalCount == 0) { throw new ArgumentException("The given predictions are empty."); } return(totalSum / totalCount); }
/// <summary> /// Computes the expected confusion matrix. /// </summary> /// <param name="instanceSource">The instance source providing the ground truth.</param> /// <param name="predictions">A sparse users-by-items matrix of predicted rating distributions.</param> /// <param name="aggregationMethod">A method specifying how metrics are aggregated over all instances.</param> /// <returns>The computed expected confusion matrix.</returns> public RatingMatrix ExpectedConfusionMatrix( TInstanceSource instanceSource, IDictionary <TUser, IDictionary <TItem, Discrete> > predictions, RecommenderMetricAggregationMethod aggregationMethod = RecommenderMetricAggregationMethod.Default) { IStarRatingInfo <TGroundTruthRating> starRatingInfo = this.mapping.GetRatingInfo(instanceSource); var result = new RatingMatrix(starRatingInfo.MinStarRating, starRatingInfo.MaxStarRating); for (int predictedRating = starRatingInfo.MinStarRating; predictedRating <= starRatingInfo.MaxStarRating; ++predictedRating) { for (int trueRating = starRatingInfo.MinStarRating; trueRating <= starRatingInfo.MaxStarRating; ++trueRating) { result[predictedRating, trueRating] = this.ModelDomainRatingPredictionMetricExpectation( instanceSource, predictions, (p, t) => p == predictedRating && t == trueRating ? 1.0 : 0.0, aggregationMethod); } } return(result); }