/// <summary>Evaluates a rating predictor for RMSE, MAE, and NMAE</summary> /// <remarks> /// For NMAE, see "Eigentaste: A Constant Time Collaborative Filtering Algorithm" by Goldberg et al. /// </remarks> /// <param name="recommender">rating predictor</param> /// <param name="ratings">Test cases</param> /// <returns>a Dictionary containing the evaluation results</returns> public static Dictionary<string, double> Evaluate(IRatingPredictor recommender, IRatings ratings) { double rmse = 0; double mae = 0; if (recommender == null) throw new ArgumentNullException("recommender"); if (ratings == null) throw new ArgumentNullException("ratings"); for (int index = 0; index < ratings.Count; index++) { double error = (recommender.Predict(ratings.Users[index], ratings.Items[index]) - ratings[index]); rmse += error * error; mae += Math.Abs(error); } mae = mae / ratings.Count; rmse = Math.Sqrt(rmse / ratings.Count); var result = new Dictionary<string, double>(); result["RMSE"] = rmse; result["MAE"] = mae; result["NMAE"] = mae / (recommender.MaxRating - recommender.MinRating); return result; }
/// <summary>Online evaluation for rating prediction</summary> /// <remarks> /// Every rating that is tested is added to the training set afterwards. /// </remarks> /// <param name="recommender">rating predictor</param> /// <param name="ratings">Test cases</param> /// <returns>a Dictionary containing the evaluation results</returns> static public Dictionary <string, double> EvaluateOnline(IRatingPredictor recommender, IRatings ratings) { double rmse = 0; double mae = 0; if (recommender == null) { throw new ArgumentNullException("recommender"); } if (ratings == null) { throw new ArgumentNullException("ratings"); } // iterate in random order // TODO also support chronological order foreach (int index in ratings.RandomIndex) { double error = (recommender.Predict(ratings.Users[index], ratings.Items[index]) - ratings[index]); rmse += error * error; mae += Math.Abs(error); recommender.AddRating(ratings.Users[index], ratings.Items[index], ratings[index]); } mae = mae / ratings.Count; rmse = Math.Sqrt(rmse / ratings.Count); var result = new Dictionary <string, double>(); result["RMSE"] = rmse; result["MAE"] = mae; result["NMAE"] = mae / (recommender.MaxRating - recommender.MinRating); return(result); }
/// <summary>Evaluates a rating predictor for RMSE, MAE, and NMAE</summary> /// <remarks> /// For NMAE, see "Eigentaste: A Constant Time Collaborative Filtering Algorithm" by Goldberg et al. /// </remarks> /// <param name="recommender">rating predictor</param> /// <param name="ratings">Test cases</param> /// <returns>a Dictionary containing the evaluation results</returns> static public Dictionary <string, double> Evaluate(IRatingPredictor recommender, IRatings ratings) { double rmse = 0; double mae = 0; if (recommender == null) { throw new ArgumentNullException("recommender"); } if (ratings == null) { throw new ArgumentNullException("ratings"); } for (int index = 0; index < ratings.Count; index++) { double error = (recommender.Predict(ratings.Users[index], ratings.Items[index]) - ratings[index]); rmse += error * error; mae += Math.Abs(error); } mae = mae / ratings.Count; rmse = Math.Sqrt(rmse / ratings.Count); var result = new Dictionary <string, double>(); result["RMSE"] = rmse; result["MAE"] = mae; result["NMAE"] = mae / (recommender.MaxRating - recommender.MinRating); return(result); }
/// <summary>Evaluates a rating predictor for RMSE, (N)MAE, and CBD</summary> /// <remarks> /// <para> /// See http://recsyswiki.com/wiki/Root_mean_square_error and http://recsyswiki.com/wiki/Mean_absolute_error /// </para> /// <para> /// For NMAE, see the paper by Goldberg et al. /// </para> /// <para> /// For CBD (capped binomial deviance), see http://www.kaggle.com/c/ChessRatings2/Details/Evaluation /// </para> /// <para> /// If the recommender can take time into account, and the rating dataset provides rating times, /// this information will be used for making rating predictions. /// </para> /// <para> /// Literature: /// <list type="bullet"> /// <item><description> /// Ken Goldberg, Theresa Roeder, Dhruv Gupta, and Chris Perkins: /// Eigentaste: A Constant Time Collaborative Filtering Algorithm. /// nformation Retrieval Journal 2001. /// http://goldberg.berkeley.edu/pubs/eigentaste.pdf /// </description></item> /// </list> /// </para> /// </remarks> /// <param name="recommender">rating predictor</param> /// <param name="test_ratings">test cases</param> /// <param name="training_ratings">the training examples</param> /// <returns>a Dictionary containing the evaluation results</returns> static public RatingPredictionEvaluationResults Evaluate(this IRatingPredictor recommender, IRatings test_ratings, IRatings training_ratings = null) { if (recommender == null) { throw new ArgumentNullException("recommender"); } if (test_ratings == null) { throw new ArgumentNullException("ratings"); } var all_indices = Enumerable.Range(0, test_ratings.Count).ToArray(); var results = new RatingPredictionEvaluationResults(Evaluate(recommender, test_ratings, all_indices)); if (training_ratings != null) { var new_user_indices = (from index in all_indices where test_ratings.Users[index] > training_ratings.MaxUserID || training_ratings.CountByUser[test_ratings.Users[index]] == 0 select index).ToArray(); results.NewUserResults = Evaluate(recommender, test_ratings, new_user_indices); var new_item_indices = (from index in all_indices where test_ratings.Items[index] > training_ratings.MaxItemID || training_ratings.CountByItem[test_ratings.Items[index]] == 0 select index).ToArray(); results.NewItemResults = Evaluate(recommender, test_ratings, new_item_indices); results.NewUserNewItemResults = Evaluate(recommender, test_ratings, Enumerable.Intersect(new_user_indices, new_item_indices).ToArray()); } return(results); }
/// <summary>Rate a given set of instances and write it to a TextWriter</summary> /// <param name="recommender">rating predictor</param> /// <param name="ratings">test cases</param> /// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param> /// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param> /// <param name="writer">the TextWriter to write the predictions to</param> public static void WritePredictions( IRatingPredictor recommender, IRatings ratings, IEntityMapping user_mapping, IEntityMapping item_mapping, TextWriter writer) { WritePredictions(recommender, ratings, user_mapping, item_mapping, "{0}\t{1}\t{2}", writer); }
/// <summary>Rate a given set of instances and write it to a file</summary> /// <param name="recommender">rating predictor</param> /// <param name="ratings">test cases</param> /// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param> /// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param> /// <param name="filename">the name of the file to write the predictions to</param> public static void WritePredictions( IRatingPredictor recommender, IRatings ratings, IEntityMapping user_mapping, IEntityMapping item_mapping, string filename) { WritePredictions(recommender, ratings, user_mapping, item_mapping, "{0}\t{1}\t{2}", filename); }
/// <summary>Rate a given set of instances and write it to a file</summary> /// <param name="recommender">rating predictor</param> /// <param name="ratings">test cases</param> /// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param> /// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param> /// <param name="filename">the name of the file to write the predictions to</param> public static void WritePredictions( IRatingPredictor recommender, IRatings ratings, IEntityMapping user_mapping, IEntityMapping item_mapping, string filename) { WritePredictions(recommender, ratings, user_mapping, item_mapping, "{0}\t{1}\t{2}", filename); }
/// <summary>Computes the absolute error sum</summary> /// <returns>the absolute error sum</returns> /// <param name='recommender'>the recommender to make predictions with</param> /// <param name='ratings'>the actual ratings</param> public static double ComputeAbsoluteErrorSum(this IRatingPredictor recommender, IRatings ratings) { double sum = 0; for (int i = 0; i < ratings.Count; i++) { sum += Math.Abs(recommender.Predict(ratings.Users[i], ratings.Items[i]) - ratings[i]); } return(sum); }
/// <summary>Rate a given set of instances and write it to a file</summary> /// <param name="recommender">rating predictor</param> /// <param name="ratings">test cases</param> /// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param> /// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param> /// <param name="line_format">a format string specifying the line format; {0} is the user ID, {1} the item ID, {2} the rating</param> /// <param name="filename">the name of the file to write the predictions to</param> public static void WritePredictions( IRatingPredictor recommender, IRatings ratings, IEntityMapping user_mapping, IEntityMapping item_mapping, string line_format, string filename) { if (filename.Equals("-")) WritePredictions(recommender, ratings, user_mapping, item_mapping, line_format, Console.Out); else using ( var writer = new StreamWriter(filename) ) WritePredictions(recommender, ratings, user_mapping, item_mapping, line_format, writer); }
// TODO as soon as we drop support for Mono 2.6, use default arguments /// <summary>Rate a given set of instances and write it to a TextWriter</summary> /// <param name="recommender">rating predictor</param> /// <param name="ratings">test cases</param> /// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param> /// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param> /// <param name="line_format">a format string specifying the line format; {0} is the user ID, {1} the item ID, {2} the rating</param> /// <param name="writer">the TextWriter to write the predictions to</param> public static void WritePredictions( IRatingPredictor recommender, IRatings ratings, IEntityMapping user_mapping, IEntityMapping item_mapping, string line_format, TextWriter writer) { for (int index = 0; index < ratings.Count; index++) writer.WriteLine(line_format, user_mapping.ToOriginalID(ratings.Users[index]), item_mapping.ToOriginalID(ratings.Items[index]), recommender.Predict(ratings.Users[index], ratings.Items[index]).ToString(CultureInfo.InvariantCulture)); }
static Dictionary <string, float> Evaluate(IRatingPredictor recommender, IRatings ratings, IList <int> indices) { if (indices.Count == 0) { return(null); } double rmse = 0; double mae = 0; double cbd = 0; if (recommender is ITimeAwareRatingPredictor && ratings is ITimedRatings) { var time_aware_recommender = recommender as ITimeAwareRatingPredictor; var timed_ratings = ratings as ITimedRatings; foreach (int index in indices) { float prediction = time_aware_recommender.Predict(timed_ratings.Users[index], timed_ratings.Items[index], timed_ratings.Times[index]); float error = prediction - ratings[index]; rmse += error * error; mae += Math.Abs(error); cbd += ComputeCBD(ratings[index], prediction, ratings.Scale.Min, ratings.Scale.Max); } } else { foreach (int index in indices) { float prediction = recommender.Predict(ratings.Users[index], ratings.Items[index]); float error = prediction - ratings[index]; rmse += error * error; mae += Math.Abs(error); cbd += ComputeCBD(ratings[index], prediction, ratings.Scale.Min, ratings.Scale.Max); } } mae = mae / indices.Count; rmse = Math.Sqrt(rmse / indices.Count); cbd = cbd / indices.Count; var result = new Dictionary <string, float>(); result["RMSE"] = (float)rmse; result["MAE"] = (float)mae; result["NMAE"] = (float)mae / (recommender.MaxRating - recommender.MinRating); result["CBD"] = (float)cbd; return(result); }
// TODO as soon as we drop support for Mono 2.6, use default arguments /// <summary>Rate a given set of instances and write it to a TextWriter</summary> /// <param name="recommender">rating predictor</param> /// <param name="ratings">test cases</param> /// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param> /// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param> /// <param name="line_format">a format string specifying the line format; {0} is the user ID, {1} the item ID, {2} the rating</param> /// <param name="writer">the TextWriter to write the predictions to</param> public static void WritePredictions( IRatingPredictor recommender, IRatings ratings, IEntityMapping user_mapping, IEntityMapping item_mapping, string line_format, TextWriter writer) { for (int index = 0; index < ratings.Count; index++) { writer.WriteLine(line_format, user_mapping.ToOriginalID(ratings.Users[index]), item_mapping.ToOriginalID(ratings.Items[index]), recommender.Predict(ratings.Users[index], ratings.Items[index]).ToString(CultureInfo.InvariantCulture)); } }
/// <summary>Rate a given set of instances and write it to a file</summary> /// <param name="recommender">rating predictor</param> /// <param name="ratings">test cases</param> /// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param> /// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param> /// <param name="line_format">a format string specifying the line format; {0} is the user ID, {1} the item ID, {2} the rating</param> /// <param name="filename">the name of the file to write the predictions to</param> public static void WritePredictions( IRatingPredictor recommender, IRatings ratings, IEntityMapping user_mapping, IEntityMapping item_mapping, string line_format, string filename) { if (filename.Equals("-")) { WritePredictions(recommender, ratings, user_mapping, item_mapping, line_format, Console.Out); } else { using (var writer = new StreamWriter(filename)) WritePredictions(recommender, ratings, user_mapping, item_mapping, line_format, writer); } }
/// <summary>Online evaluation for rating prediction</summary> /// <remarks> /// Every rating that is tested is added to the training set afterwards. /// </remarks> /// <param name="recommender">rating predictor</param> /// <param name="ratings">Test cases</param> /// <returns>a Dictionary containing the evaluation results</returns> static public RatingPredictionEvaluationResults EvaluateOnline(this IRatingPredictor recommender, IRatings ratings) { if (recommender == null) throw new ArgumentNullException("recommender"); if (ratings == null) throw new ArgumentNullException("ratings"); var incremental_recommender = recommender as IIncrementalRatingPredictor; if (incremental_recommender == null) throw new ArgumentException("recommender must be of type IIncrementalRatingPredictor"); double rmse = 0; double mae = 0; double cbd = 0; // iterate in random order foreach (int index in ratings.RandomIndex) { float prediction = recommender.Predict(ratings.Users[index], ratings.Items[index]); float error = prediction - ratings[index]; rmse += error * error; mae += Math.Abs(error); cbd += Eval.Ratings.ComputeCBD(ratings[index], prediction, recommender.MinRating, recommender.MaxRating); incremental_recommender.AddRatings(new RatingsProxy(ratings, new int[] { index })); } mae = mae / ratings.Count; rmse = Math.Sqrt(rmse / ratings.Count); cbd = cbd / ratings.Count; var result = new RatingPredictionEvaluationResults(); result["RMSE"] = (float) rmse; result["MAE"] = (float) mae; result["NMAE"] = (float) mae / (recommender.MaxRating - recommender.MinRating); result["CBD"] = (float) cbd; return result; }
/// <summary>Computes the logistic loss sum</summary> /// <returns>the logistic loss sum</returns> /// <param name='recommender'>the recommender to make predictions with</param> /// <param name='ratings'>the actual ratings</param> /// <param name='min_rating'>the minimal rating</param> /// <param name='rating_range_size'>the size of the rating range: max_rating - min_rating</param> public static double ComputeSum( this IRatingPredictor recommender, IRatings ratings, float min_rating, float rating_range_size) { double sum = 0; for (int i = 0; i < ratings.Count; i++) { double prediction = recommender.Predict(ratings.Users[i], ratings.Items[i]); // map into [0, 1] interval prediction = (prediction - min_rating) / rating_range_size; if (prediction < 0.0) prediction = 0.0; if (prediction > 1.0) prediction = 1.0; double actual_rating = (ratings[i] - min_rating) / rating_range_size; sum -= (actual_rating) * Math.Log(prediction); sum -= (1 - actual_rating) * Math.Log(1 - prediction); } return sum; }
/// <summary>Rate a given set of instances and write it to a TextWriter</summary> /// <param name="recommender">rating predictor</param> /// <param name="ratings">test cases</param> /// <param name="user_mapping">an <see cref="EntityMapping"/> object for the user IDs</param> /// <param name="item_mapping">an <see cref="EntityMapping"/> object for the item IDs</param> /// <param name="writer">the TextWriter to write the predictions to</param> public static void WritePredictions( IRatingPredictor recommender, IRatings ratings, IEntityMapping user_mapping, IEntityMapping item_mapping, TextWriter writer) { WritePredictions(recommender, ratings, user_mapping, item_mapping, "{0}\t{1}\t{2}", writer); }
/// <summary>Online evaluation for rating prediction</summary> /// <remarks> /// Every rating that is tested is added to the training set afterwards. /// </remarks> /// <param name="recommender">rating predictor</param> /// <param name="ratings">Test cases</param> /// <returns>a Dictionary containing the evaluation results</returns> public static Dictionary<string, double> EvaluateOnline(IRatingPredictor recommender, IRatings ratings) { double rmse = 0; double mae = 0; if (recommender == null) throw new ArgumentNullException("recommender"); if (ratings == null) throw new ArgumentNullException("ratings"); // iterate in random order // TODO also support chronological order foreach (int index in ratings.RandomIndex) { double error = (recommender.Predict(ratings.Users[index], ratings.Items[index]) - ratings[index]); rmse += error * error; mae += Math.Abs(error); recommender.AddRating(ratings.Users[index], ratings.Items[index], ratings[index]); } mae = mae / ratings.Count; rmse = Math.Sqrt(rmse / ratings.Count); var result = new Dictionary<string, double>(); result["RMSE"] = rmse; result["MAE"] = mae; result["NMAE"] = mae / (recommender.MaxRating - recommender.MinRating); return result; }
public MediaLiteRatingPredictor(MyMediaLite.IRecommender recommender) { _ratingPredictor = recommender as MyMediaLite.RatingPrediction.IRatingPredictor; _usersMap = new Mapping(); _itemsMap = new Mapping(); }
static Dictionary<string, float> Evaluate(IRatingPredictor recommender, IRatings ratings, IList<int> indices) { if (indices.Count == 0) return null; double rmse = 0; double mae = 0; double cbd = 0; if (recommender is ITimeAwareRatingPredictor && ratings is ITimedRatings) { var time_aware_recommender = recommender as ITimeAwareRatingPredictor; var timed_ratings = ratings as ITimedRatings; foreach (int index in indices) { float prediction = time_aware_recommender.Predict(timed_ratings.Users[index], timed_ratings.Items[index], timed_ratings.Times[index]); float error = prediction - ratings[index]; rmse += error * error; mae += Math.Abs(error); cbd += ComputeCBD(ratings[index], prediction, ratings.Scale.Min, ratings.Scale.Max); } } else foreach (int index in indices) { float prediction = recommender.Predict(ratings.Users[index], ratings.Items[index]); float error = prediction - ratings[index]; rmse += error * error; mae += Math.Abs(error); cbd += ComputeCBD(ratings[index], prediction, ratings.Scale.Min, ratings.Scale.Max); } mae = mae / indices.Count; rmse = Math.Sqrt(rmse / indices.Count); cbd = cbd / indices.Count; var result = new Dictionary<string, float>(); result["RMSE"] = (float) rmse; result["MAE"] = (float) mae; result["NMAE"] = (float) mae / (recommender.MaxRating - recommender.MinRating); result["CBD"] = (float) cbd; return result; }