/// <summary>
/// Computes the average of a given data domain rating prediction metric by iterating firstly per-item and then per-user.
/// </summary>
/// <param name="instanceSource">The source of the instances providing the ground truth.</param>
/// <param name="predictions">A sparse users-by-items matrix of predicted rating distributions.</param>
/// <param name="metric">The data domain rating prediction metric to average.</param>
/// <param name="aggregationMethod">A method specifying how metrics are aggregated over the whole dataset.</param>
/// <returns>The computed average of the given rating prediction metric.</returns>
public double RatingPredictionMetric(
TInstanceSource instanceSource,
IDictionary <TUser, IDictionary <TItem, TPredictedRatingDist> > predictions,
Func <TGroundTruthRating, TPredictedRatingDist, double> metric,
RecommenderMetricAggregationMethod aggregationMethod = RecommenderMetricAggregationMethod.Default)
{
return(this.RatingPredictionMetric <TPredictedRatingDist>(instanceSource, predictions, metric, aggregationMethod));
}
/// <summary>
/// Computes the expected component-wise product of a confusion matrix and a loss matrix.
/// </summary>
/// <param name="instanceSource">The instance source providing the ground truth (used to compute the expected confusion matrix).</param>
/// <param name="predictions">
/// A sparse users-by-items matrix of predicted rating distributions (used to compute the expected confusion matrix).
/// </param>
/// <param name="lossMatrix">The loss matrix.</param>
/// <param name="aggregationMethod">
/// A method specifying how metrics are aggregated over all instances (used to compute the expected confusion matrix).</param>
/// <returns>The computed expected weighted confusion.</returns>
public double ExpectedWeightedConfusion(
TInstanceSource instanceSource,
IDictionary <TUser, IDictionary <TItem, Discrete> > predictions,
RatingMatrix lossMatrix,
RecommenderMetricAggregationMethod aggregationMethod = RecommenderMetricAggregationMethod.Default)
{
RatingMatrix confusionMatrix = this.ExpectedConfusionMatrix(instanceSource, predictions, aggregationMethod);
return(RatingMatrix.ComponentwiseProduct(confusionMatrix, lossMatrix));
}
/// <summary>
/// Computes the average of a given rating prediction metric using ground truth in model domain by iterating over
/// <paramref name="predictions"/> and using the aggregation method given in <paramref name="aggregationMethod"/>.
/// </summary>
/// <param name="instanceSource">The instance source providing the ground truth.</param>
/// <param name="predictions">A sparse users-by-items matrix of predicted rating distributions.</param>
/// <param name="metric">The rating prediction metric using ground truth in model domain.</param>
/// <param name="aggregationMethod">A method specifying how metrics are aggregated over all instances.</param>
/// <returns>The computed average of the given rating prediction metric.</returns>
public double ModelDomainRatingPredictionMetric(
TInstanceSource instanceSource,
IDictionary <TUser, IDictionary <TItem, Discrete> > predictions,
Func <int, Discrete, double> metric,
RecommenderMetricAggregationMethod aggregationMethod = RecommenderMetricAggregationMethod.Default)
{
IStarRatingInfo <TGroundTruthRating> starRatingInfo = this.mapping.GetRatingInfo(instanceSource);
Func <TGroundTruthRating, Discrete, double> metricWrapper = (g, up) => metric(starRatingInfo.ToStarRating(g), up);
return(this.RatingPredictionMetric(instanceSource, predictions, metricWrapper, aggregationMethod));
}
/// <summary>
/// Computes the average of a given rating prediction metric using ground truth in model domain by iterating over
/// <paramref name="predictions"/> and using the aggregation method given in <paramref name="aggregationMethod"/>.
/// </summary>
/// <param name="instanceSource">The instance source providing the ground truth.</param>
/// <param name="predictions">A sparse users-by-items matrix of predicted rating distributions.</param>
/// <param name="metric">The rating prediction metric using ground truth in model domain.</param>
/// <param name="aggregationMethod">A method specifying how metrics are aggregated over all instances.</param>
/// <returns>The computed average of the given rating prediction metric.</returns>
public double ModelDomainRatingPredictionMetricExpectation(
TInstanceSource instanceSource,
IDictionary <TUser, IDictionary <TItem, Discrete> > predictions,
Func <int, int, double> metric,
RecommenderMetricAggregationMethod aggregationMethod = RecommenderMetricAggregationMethod.Default)
{
return(this.ModelDomainRatingPredictionMetric(
instanceSource,
predictions,
(r, ur) => RatingMetricExpectation(r, ur, metric),
aggregationMethod));
}
/// <summary>
/// Computes the average of a given rating prediction metric by iterating over
/// <paramref name="predictions"/> and using the aggregation method given in <paramref name="aggregationMethod"/>.
/// </summary>
/// <typeparam name="TPrediction">
/// The type of predictions
/// (used to support both certain and uncertain predictions).
/// </typeparam>
/// <param name="instanceSource">The source of the instances providing the ground truth.</param>
/// <param name="predictions">A sparse users-by-items matrix of predicted rating distributions.</param>
/// <param name="metric">The rating prediction metric to average.</param>
/// <param name="aggregationMethod">A method specifying how metrics are aggregated over the whole dataset.</param>
/// <returns>The computed average of the given rating prediction metric.</returns>
private double RatingPredictionMetric <TPrediction>(
TInstanceSource instanceSource,
IDictionary <TUser, IDictionary <TItem, TPrediction> > predictions,
Func <TGroundTruthRating, TPrediction, double> metric,
RecommenderMetricAggregationMethod aggregationMethod)
{
double totalSum = 0;
int totalCount = 0;
foreach (var userWithPredictionList in predictions)
{
double perUserSum = 0;
int perUserCount = 0;
foreach (var itemPrediction in userWithPredictionList.Value)
{
TPrediction prediction = itemPrediction.Value;
TGroundTruthRating groundTruth = this.mapping.GetRating(instanceSource, userWithPredictionList.Key, itemPrediction.Key);
double metricValue = metric(groundTruth, prediction);
perUserSum += metricValue;
++perUserCount;
}
switch (aggregationMethod)
{
case RecommenderMetricAggregationMethod.PerUserFirst:
if (perUserCount == 0)
{
throw new ArgumentException("One of the users in the given predictions has no associated predictions.");
}
totalSum += perUserSum / perUserCount;
++totalCount;
break;
case RecommenderMetricAggregationMethod.Default:
totalSum += perUserSum;
totalCount += perUserCount;
break;
default:
Debug.Fail("Metric aggregation method is not supported.");
break;
}
}
if (totalCount == 0)
{
throw new ArgumentException("The given predictions are empty.");
}
return(totalSum / totalCount);
}
/// <summary>
/// Computes the expected confusion matrix.
/// </summary>
/// <param name="instanceSource">The instance source providing the ground truth.</param>
/// <param name="predictions">A sparse users-by-items matrix of predicted rating distributions.</param>
/// <param name="aggregationMethod">A method specifying how metrics are aggregated over all instances.</param>
/// <returns>The computed expected confusion matrix.</returns>
public RatingMatrix ExpectedConfusionMatrix(
TInstanceSource instanceSource,
IDictionary <TUser, IDictionary <TItem, Discrete> > predictions,
RecommenderMetricAggregationMethod aggregationMethod = RecommenderMetricAggregationMethod.Default)
{
IStarRatingInfo <TGroundTruthRating> starRatingInfo = this.mapping.GetRatingInfo(instanceSource);
var result = new RatingMatrix(starRatingInfo.MinStarRating, starRatingInfo.MaxStarRating);
for (int predictedRating = starRatingInfo.MinStarRating; predictedRating <= starRatingInfo.MaxStarRating; ++predictedRating)
{
for (int trueRating = starRatingInfo.MinStarRating; trueRating <= starRatingInfo.MaxStarRating; ++trueRating)
{
result[predictedRating, trueRating] = this.ModelDomainRatingPredictionMetricExpectation(
instanceSource,
predictions,
(p, t) => p == predictedRating && t == trueRating ? 1.0 : 0.0,
aggregationMethod);
}
}
return(result);
}
