/// <summary>Evaluates a rating predictor for RMSE, MAE, and NMAE</summary>
/// <remarks>
/// For NMAE, see "Eigentaste: A Constant Time Collaborative Filtering Algorithm" by Goldberg et al.
/// </remarks>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">Test cases</param>
/// <returns>a Dictionary containing the evaluation results</returns>
public static Dictionary<string, double> Evaluate(IRatingPredictor recommender, IRatings ratings)
{
double rmse = 0;
double mae = 0;
if (recommender == null)
throw new ArgumentNullException("recommender");
if (ratings == null)
throw new ArgumentNullException("ratings");
for (int index = 0; index < ratings.Count; index++)
{
double error = (recommender.Predict(ratings.Users[index], ratings.Items[index]) - ratings[index]);
rmse += error * error;
mae += Math.Abs(error);
}
mae = mae / ratings.Count;
rmse = Math.Sqrt(rmse / ratings.Count);
var result = new Dictionary<string, double>();
result["RMSE"] = rmse;
result["MAE"] = mae;
result["NMAE"] = mae / (recommender.MaxRating - recommender.MinRating);
return result;
}
/// <summary>Online evaluation for rating prediction</summary>
/// <remarks>
/// Every rating that is tested is added to the training set afterwards.
/// </remarks>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">Test cases</param>
/// <returns>a Dictionary containing the evaluation results</returns>
static public Dictionary <string, double> EvaluateOnline(IRatingPredictor recommender, IRatings ratings)
{
double rmse = 0;
double mae = 0;
if (recommender == null)
{
throw new ArgumentNullException("recommender");
}
if (ratings == null)
{
throw new ArgumentNullException("ratings");
}
// iterate in random order // TODO also support chronological order
foreach (int index in ratings.RandomIndex)
{
double error = (recommender.Predict(ratings.Users[index], ratings.Items[index]) - ratings[index]);
rmse += error * error;
mae += Math.Abs(error);
recommender.AddRating(ratings.Users[index], ratings.Items[index], ratings[index]);
}
mae = mae / ratings.Count;
rmse = Math.Sqrt(rmse / ratings.Count);
var result = new Dictionary <string, double>();
result["RMSE"] = rmse;
result["MAE"] = mae;
result["NMAE"] = mae / (recommender.MaxRating - recommender.MinRating);
return(result);
}
/// <summary>Evaluates a rating predictor for RMSE, MAE, and NMAE</summary>
/// <remarks>
/// For NMAE, see "Eigentaste: A Constant Time Collaborative Filtering Algorithm" by Goldberg et al.
/// </remarks>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">Test cases</param>
/// <returns>a Dictionary containing the evaluation results</returns>
static public Dictionary <string, double> Evaluate(IRatingPredictor recommender, IRatings ratings)
{
double rmse = 0;
double mae = 0;
if (recommender == null)
{
throw new ArgumentNullException("recommender");
}
if (ratings == null)
{
throw new ArgumentNullException("ratings");
}
for (int index = 0; index < ratings.Count; index++)
{
double error = (recommender.Predict(ratings.Users[index], ratings.Items[index]) - ratings[index]);
rmse += error * error;
mae += Math.Abs(error);
}
mae = mae / ratings.Count;
rmse = Math.Sqrt(rmse / ratings.Count);
var result = new Dictionary <string, double>();
result["RMSE"] = rmse;
result["MAE"] = mae;
result["NMAE"] = mae / (recommender.MaxRating - recommender.MinRating);
return(result);
}
/// <summary>Evaluates a rating predictor for RMSE, (N)MAE, and CBD</summary>
/// <remarks>
/// <para>
/// See http://recsyswiki.com/wiki/Root_mean_square_error and http://recsyswiki.com/wiki/Mean_absolute_error
/// </para>
/// <para>
/// For NMAE, see the paper by Goldberg et al.
/// </para>
/// <para>
/// For CBD (capped binomial deviance), see http://www.kaggle.com/c/ChessRatings2/Details/Evaluation
/// </para>
/// <para>
/// If the recommender can take time into account, and the rating dataset provides rating times,
/// this information will be used for making rating predictions.
/// </para>
/// <para>
/// Literature:
/// <list type="bullet">
/// <item><description>
/// Ken Goldberg, Theresa Roeder, Dhruv Gupta, and Chris Perkins:
/// Eigentaste: A Constant Time Collaborative Filtering Algorithm.
/// nformation Retrieval Journal 2001.
/// http://goldberg.berkeley.edu/pubs/eigentaste.pdf
/// </description></item>
/// </list>
/// </para>
/// </remarks>
/// <param name="recommender">rating predictor</param>
/// <param name="test_ratings">test cases</param>
/// <param name="training_ratings">the training examples</param>
/// <returns>a Dictionary containing the evaluation results</returns>
static public RatingPredictionEvaluationResults Evaluate(this IRatingPredictor recommender, IRatings test_ratings, IRatings training_ratings = null)
{
if (recommender == null)
{
throw new ArgumentNullException("recommender");
}
if (test_ratings == null)
{
throw new ArgumentNullException("ratings");
}
var all_indices = Enumerable.Range(0, test_ratings.Count).ToArray();
var results = new RatingPredictionEvaluationResults(Evaluate(recommender, test_ratings, all_indices));
if (training_ratings != null)
{
var new_user_indices = (from index in all_indices
where test_ratings.Users[index] > training_ratings.MaxUserID || training_ratings.CountByUser[test_ratings.Users[index]] == 0
select index).ToArray();
results.NewUserResults = Evaluate(recommender, test_ratings, new_user_indices);
var new_item_indices = (from index in all_indices
where test_ratings.Items[index] > training_ratings.MaxItemID || training_ratings.CountByItem[test_ratings.Items[index]] == 0 select index).ToArray();
results.NewItemResults = Evaluate(recommender, test_ratings, new_item_indices);
results.NewUserNewItemResults = Evaluate(recommender, test_ratings, Enumerable.Intersect(new_user_indices, new_item_indices).ToArray());
}
return(results);
}
/// <summary>Rate a given set of instances and write it to a TextWriter</summary>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">test cases</param>
/// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param>
/// <param name="writer">the TextWriter to write the predictions to</param>
public static void WritePredictions(
IRatingPredictor recommender,
IRatings ratings,
IEntityMapping user_mapping, IEntityMapping item_mapping,
TextWriter writer)
{
WritePredictions(recommender, ratings, user_mapping, item_mapping, "{0}\t{1}\t{2}", writer);
}
/// <summary>Rate a given set of instances and write it to a file</summary>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">test cases</param>
/// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param>
/// <param name="filename">the name of the file to write the predictions to</param>
public static void WritePredictions(
IRatingPredictor recommender,
IRatings ratings,
IEntityMapping user_mapping, IEntityMapping item_mapping,
string filename)
{
WritePredictions(recommender, ratings, user_mapping, item_mapping, "{0}\t{1}\t{2}", filename);
}
/// <summary>Rate a given set of instances and write it to a file</summary>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">test cases</param>
/// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param>
/// <param name="filename">the name of the file to write the predictions to</param>
public static void WritePredictions(
IRatingPredictor recommender,
IRatings ratings,
IEntityMapping user_mapping, IEntityMapping item_mapping,
string filename)
{
WritePredictions(recommender, ratings, user_mapping, item_mapping, "{0}\t{1}\t{2}", filename);
}
/// <summary>Computes the absolute error sum</summary>
/// <returns>the absolute error sum</returns>
/// <param name='recommender'>the recommender to make predictions with</param>
/// <param name='ratings'>the actual ratings</param>
public static double ComputeAbsoluteErrorSum(this IRatingPredictor recommender, IRatings ratings)
{
double sum = 0;
for (int i = 0; i < ratings.Count; i++)
{
sum += Math.Abs(recommender.Predict(ratings.Users[i], ratings.Items[i]) - ratings[i]);
}
return(sum);
}
/// <summary>Rate a given set of instances and write it to a file</summary>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">test cases</param>
/// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param>
/// <param name="line_format">a format string specifying the line format; {0} is the user ID, {1} the item ID, {2} the rating</param>
/// <param name="filename">the name of the file to write the predictions to</param>
public static void WritePredictions(
IRatingPredictor recommender,
IRatings ratings,
IEntityMapping user_mapping, IEntityMapping item_mapping,
string line_format,
string filename)
{
if (filename.Equals("-"))
WritePredictions(recommender, ratings, user_mapping, item_mapping, line_format, Console.Out);
else
using ( var writer = new StreamWriter(filename) )
WritePredictions(recommender, ratings, user_mapping, item_mapping, line_format, writer);
}
// TODO as soon as we drop support for Mono 2.6, use default arguments
/// <summary>Rate a given set of instances and write it to a TextWriter</summary>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">test cases</param>
/// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param>
/// <param name="line_format">a format string specifying the line format; {0} is the user ID, {1} the item ID, {2} the rating</param>
/// <param name="writer">the TextWriter to write the predictions to</param>
public static void WritePredictions(
IRatingPredictor recommender,
IRatings ratings,
IEntityMapping user_mapping, IEntityMapping item_mapping,
string line_format,
TextWriter writer)
{
for (int index = 0; index < ratings.Count; index++)
writer.WriteLine(line_format,
user_mapping.ToOriginalID(ratings.Users[index]),
item_mapping.ToOriginalID(ratings.Items[index]),
recommender.Predict(ratings.Users[index], ratings.Items[index]).ToString(CultureInfo.InvariantCulture));
}
static Dictionary <string, float> Evaluate(IRatingPredictor recommender, IRatings ratings, IList <int> indices)
{
if (indices.Count == 0)
{
return(null);
}
double rmse = 0;
double mae = 0;
double cbd = 0;
if (recommender is ITimeAwareRatingPredictor && ratings is ITimedRatings)
{
var time_aware_recommender = recommender as ITimeAwareRatingPredictor;
var timed_ratings = ratings as ITimedRatings;
foreach (int index in indices)
{
float prediction = time_aware_recommender.Predict(timed_ratings.Users[index], timed_ratings.Items[index], timed_ratings.Times[index]);
float error = prediction - ratings[index];
rmse += error * error;
mae += Math.Abs(error);
cbd += ComputeCBD(ratings[index], prediction, ratings.Scale.Min, ratings.Scale.Max);
}
}
else
{
foreach (int index in indices)
{
float prediction = recommender.Predict(ratings.Users[index], ratings.Items[index]);
float error = prediction - ratings[index];
rmse += error * error;
mae += Math.Abs(error);
cbd += ComputeCBD(ratings[index], prediction, ratings.Scale.Min, ratings.Scale.Max);
}
}
mae = mae / indices.Count;
rmse = Math.Sqrt(rmse / indices.Count);
cbd = cbd / indices.Count;
var result = new Dictionary <string, float>();
result["RMSE"] = (float)rmse;
result["MAE"] = (float)mae;
result["NMAE"] = (float)mae / (recommender.MaxRating - recommender.MinRating);
result["CBD"] = (float)cbd;
return(result);
}
// TODO as soon as we drop support for Mono 2.6, use default arguments
/// <summary>Rate a given set of instances and write it to a TextWriter</summary>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">test cases</param>
/// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param>
/// <param name="line_format">a format string specifying the line format; {0} is the user ID, {1} the item ID, {2} the rating</param>
/// <param name="writer">the TextWriter to write the predictions to</param>
public static void WritePredictions(
IRatingPredictor recommender,
IRatings ratings,
IEntityMapping user_mapping, IEntityMapping item_mapping,
string line_format,
TextWriter writer)
{
for (int index = 0; index < ratings.Count; index++)
{
writer.WriteLine(line_format,
user_mapping.ToOriginalID(ratings.Users[index]),
item_mapping.ToOriginalID(ratings.Items[index]),
recommender.Predict(ratings.Users[index], ratings.Items[index]).ToString(CultureInfo.InvariantCulture));
}
}
/// <summary>Rate a given set of instances and write it to a file</summary>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">test cases</param>
/// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param>
/// <param name="line_format">a format string specifying the line format; {0} is the user ID, {1} the item ID, {2} the rating</param>
/// <param name="filename">the name of the file to write the predictions to</param>
public static void WritePredictions(
IRatingPredictor recommender,
IRatings ratings,
IEntityMapping user_mapping, IEntityMapping item_mapping,
string line_format,
string filename)
{
if (filename.Equals("-"))
{
WritePredictions(recommender, ratings, user_mapping, item_mapping, line_format, Console.Out);
}
else
{
using (var writer = new StreamWriter(filename))
WritePredictions(recommender, ratings, user_mapping, item_mapping, line_format, writer);
}
}
/// <summary>Online evaluation for rating prediction</summary>
/// <remarks>
/// Every rating that is tested is added to the training set afterwards.
/// </remarks>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">Test cases</param>
/// <returns>a Dictionary containing the evaluation results</returns>
static public RatingPredictionEvaluationResults EvaluateOnline(this IRatingPredictor recommender, IRatings ratings)
{
if (recommender == null)
throw new ArgumentNullException("recommender");
if (ratings == null)
throw new ArgumentNullException("ratings");
var incremental_recommender = recommender as IIncrementalRatingPredictor;
if (incremental_recommender == null)
throw new ArgumentException("recommender must be of type IIncrementalRatingPredictor");
double rmse = 0;
double mae = 0;
double cbd = 0;
// iterate in random order
foreach (int index in ratings.RandomIndex)
{
float prediction = recommender.Predict(ratings.Users[index], ratings.Items[index]);
float error = prediction - ratings[index];
rmse += error * error;
mae += Math.Abs(error);
cbd += Eval.Ratings.ComputeCBD(ratings[index], prediction, recommender.MinRating, recommender.MaxRating);
incremental_recommender.AddRatings(new RatingsProxy(ratings, new int[] { index }));
}
mae = mae / ratings.Count;
rmse = Math.Sqrt(rmse / ratings.Count);
cbd = cbd / ratings.Count;
var result = new RatingPredictionEvaluationResults();
result["RMSE"] = (float) rmse;
result["MAE"] = (float) mae;
result["NMAE"] = (float) mae / (recommender.MaxRating - recommender.MinRating);
result["CBD"] = (float) cbd;
return result;
}
/// <summary>Computes the logistic loss sum</summary>
/// <returns>the logistic loss sum</returns>
/// <param name='recommender'>the recommender to make predictions with</param>
/// <param name='ratings'>the actual ratings</param>
/// <param name='min_rating'>the minimal rating</param>
/// <param name='rating_range_size'>the size of the rating range: max_rating - min_rating</param>
public static double ComputeSum(
this IRatingPredictor recommender,
IRatings ratings,
float min_rating, float rating_range_size)
{
double sum = 0;
for (int i = 0; i < ratings.Count; i++)
{
double prediction = recommender.Predict(ratings.Users[i], ratings.Items[i]);
// map into [0, 1] interval
prediction = (prediction - min_rating) / rating_range_size;
if (prediction < 0.0)
prediction = 0.0;
if (prediction > 1.0)
prediction = 1.0;
double actual_rating = (ratings[i] - min_rating) / rating_range_size;
sum -= (actual_rating) * Math.Log(prediction);
sum -= (1 - actual_rating) * Math.Log(1 - prediction);
}
return sum;
}
/// <summary>Rate a given set of instances and write it to a TextWriter</summary>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">test cases</param>
/// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param>
/// <param name="writer">the TextWriter to write the predictions to</param>
public static void WritePredictions(
IRatingPredictor recommender,
IRatings ratings,
IEntityMapping user_mapping, IEntityMapping item_mapping,
TextWriter writer)
{
WritePredictions(recommender, ratings, user_mapping, item_mapping, "{0}\t{1}\t{2}", writer);
}
/// <summary>Online evaluation for rating prediction</summary>
/// <remarks>
/// Every rating that is tested is added to the training set afterwards.
/// </remarks>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">Test cases</param>
/// <returns>a Dictionary containing the evaluation results</returns>
public static Dictionary<string, double> EvaluateOnline(IRatingPredictor recommender, IRatings ratings)
{
double rmse = 0;
double mae = 0;
if (recommender == null)
throw new ArgumentNullException("recommender");
if (ratings == null)
throw new ArgumentNullException("ratings");
// iterate in random order // TODO also support chronological order
foreach (int index in ratings.RandomIndex)
{
double error = (recommender.Predict(ratings.Users[index], ratings.Items[index]) - ratings[index]);
rmse += error * error;
mae += Math.Abs(error);
recommender.AddRating(ratings.Users[index], ratings.Items[index], ratings[index]);
}
mae = mae / ratings.Count;
rmse = Math.Sqrt(rmse / ratings.Count);
var result = new Dictionary<string, double>();
result["RMSE"] = rmse;
result["MAE"] = mae;
result["NMAE"] = mae / (recommender.MaxRating - recommender.MinRating);
return result;
}
public MediaLiteRatingPredictor(MyMediaLite.IRecommender recommender)
{
_ratingPredictor = recommender as MyMediaLite.RatingPrediction.IRatingPredictor;
_usersMap = new Mapping();
_itemsMap = new Mapping();
}
static Dictionary<string, float> Evaluate(IRatingPredictor recommender, IRatings ratings, IList<int> indices)
{
if (indices.Count == 0)
return null;
double rmse = 0;
double mae = 0;
double cbd = 0;
if (recommender is ITimeAwareRatingPredictor && ratings is ITimedRatings)
{
var time_aware_recommender = recommender as ITimeAwareRatingPredictor;
var timed_ratings = ratings as ITimedRatings;
foreach (int index in indices)
{
float prediction = time_aware_recommender.Predict(timed_ratings.Users[index], timed_ratings.Items[index], timed_ratings.Times[index]);
float error = prediction - ratings[index];
rmse += error * error;
mae += Math.Abs(error);
cbd += ComputeCBD(ratings[index], prediction, ratings.Scale.Min, ratings.Scale.Max);
}
}
else
foreach (int index in indices)
{
float prediction = recommender.Predict(ratings.Users[index], ratings.Items[index]);
float error = prediction - ratings[index];
rmse += error * error;
mae += Math.Abs(error);
cbd += ComputeCBD(ratings[index], prediction, ratings.Scale.Min, ratings.Scale.Max);
}
mae = mae / indices.Count;
rmse = Math.Sqrt(rmse / indices.Count);
cbd = cbd / indices.Count;
var result = new Dictionary<string, float>();
result["RMSE"] = (float) rmse;
result["MAE"] = (float) mae;
result["NMAE"] = (float) mae / (recommender.MaxRating - recommender.MinRating);
result["CBD"] = (float) cbd;
return result;
}
