/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <param name="show_results">set to true to print results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> static public Dictionary <string, double> EvaluateOnSplit(RatingPredictor recommender, ISplit <IRatings> split, bool show_results) { var avg_results = new Dictionary <string, double>(); foreach (var key in Measures) { avg_results[key] = 0; } for (int i = 0; i < split.NumberOfFolds; i++) { var split_recommender = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender split_recommender.Ratings = split.Train[i]; split_recommender.Train(); var fold_results = Evaluate(split_recommender, split.Test[i]); foreach (var key in fold_results.Keys) { avg_results[key] += fold_results[key]; } if (show_results) { Console.Error.WriteLine("fold {0}, RMSE {1,0:0.#####}, MAE {2,0:0.#####}", i, fold_results["RMSE"].ToString(CultureInfo.InvariantCulture), fold_results["MAE"].ToString(CultureInfo.InvariantCulture)); } } foreach (var key in avg_results.Keys.ToList()) { avg_results[key] /= split.NumberOfFolds; } return(avg_results); }
/// <summary>Performs user-wise fold-in evaluation, but instead of folding in perform a complete re-training with the new data</summary> /// <remarks> /// This method can be quite slow. /// </remarks> /// <returns>the evaluation results</returns> /// <param name='recommender'>a rating predictor capable of performing a user fold-in</param> /// <param name='update_data'>the rating data used to represent the users</param> /// <param name='eval_data'>the evaluation data</param> static public RatingPredictionEvaluationResults EvaluateFoldInCompleteRetraining(this RatingPredictor recommender, IRatings update_data, IRatings eval_data) { double rmse = 0; double mae = 0; double cbd = 0; int rating_count = 0; foreach (int user_id in update_data.AllUsers) { if (eval_data.AllUsers.Contains(user_id)) { var local_recommender = (RatingPredictor)recommender.Clone(); var known_ratings = new RatingsProxy(update_data, update_data.ByUser[user_id]); local_recommender.Ratings = new CombinedRatings(recommender.Ratings, known_ratings); local_recommender.Train(); var items_to_rate = (from index in eval_data.ByUser[user_id] select eval_data.Items[index]).ToArray(); var predicted_ratings = recommender.Recommend(user_id, candidate_items: items_to_rate); foreach (var pred in predicted_ratings) { float prediction = pred.Item2; float actual_rating = eval_data.Get(user_id, pred.Item1, eval_data.ByUser[user_id]); float error = prediction - actual_rating; rmse += error * error; mae += Math.Abs(error); cbd += Eval.Ratings.ComputeCBD(actual_rating, prediction, recommender.MinRating, recommender.MaxRating); rating_count++; } Console.Error.Write("."); } } mae = mae / rating_count; rmse = Math.Sqrt(rmse / rating_count); cbd = cbd / rating_count; var result = new RatingPredictionEvaluationResults(); result["RMSE"] = (float)rmse; result["MAE"] = (float)mae; result["NMAE"] = (float)mae / (recommender.MaxRating - recommender.MinRating); result["CBD"] = (float)cbd; return(result); }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <param name="compute_fit">if set to true measure fit on the training data as well</param> /// <param name="show_fold_results">set to true to print per-fold results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> static public RatingPredictionEvaluationResults DoCrossValidation( this RatingPredictor recommender, ISplit <IRatings> split, bool compute_fit = false, bool show_fold_results = false) { var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds]; Parallel.For(0, (int)split.NumberOfFolds, i => { try { var split_recommender = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender split_recommender.Ratings = split.Train[i]; if (recommender is ITransductiveRatingPredictor) { ((ITransductiveRatingPredictor)split_recommender).AdditionalFeedback = split.Test[i]; } split_recommender.Train(); fold_results[i] = Ratings.Evaluate(split_recommender, split.Test[i]); if (compute_fit) { fold_results[i]["fit"] = (float)split_recommender.ComputeFit(); } if (show_fold_results) { Console.Error.WriteLine("fold {0} {1}", i, fold_results[i]); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); return(new RatingPredictionEvaluationResults(fold_results)); }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">a rating predictor</param> /// <param name="split">a rating dataset split</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> static public void DoIterativeCrossValidation( this RatingPredictor recommender, ISplit <IRatings> split, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { if (!(recommender is IIterativeModel)) { throw new ArgumentException("recommender must be of type IIterativeModel"); } var split_recommenders = new RatingPredictor[split.NumberOfFolds]; var iterative_recommenders = new IIterativeModel[split.NumberOfFolds]; var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds]; // initial training and evaluation Parallel.For(0, (int)split.NumberOfFolds, i => { try { split_recommenders[i] = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender split_recommenders[i].Ratings = split.Train[i]; if (recommender is ITransductiveRatingPredictor) { ((ITransductiveRatingPredictor)split_recommenders[i]).AdditionalFeedback = split.Test[i]; } split_recommenders[i].Train(); iterative_recommenders[i] = (IIterativeModel)split_recommenders[i]; fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]); if (show_fold_results) { Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], iterative_recommenders[i].NumIter); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), iterative_recommenders[0].NumIter); // iterative training and evaluation for (int it = (int)iterative_recommenders[0].NumIter + 1; it <= max_iter; it++) { Parallel.For(0, (int)split.NumberOfFolds, i => { try { iterative_recommenders[i].Iterate(); if (it % find_iter == 0) { fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]); if (show_fold_results) { Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], it); } } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), it); } }
/// <summary>Evaluate on the folds of a dataset split</summary> /// <param name="recommender">an item recommender</param> /// <param name="split">a dataset split</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="compute_fit">if set to true measure fit on the training data as well</param> /// <param name="show_results">set to true to print results to STDERR</param> /// <returns>a dictionary containing the average results over the different folds of the split</returns> static public EvaluationResults DoRatingBasedRankingCrossValidation( this RatingPredictor recommender, ISplit <IRatings> split, IList <int> candidate_items, CandidateItems candidate_item_mode = CandidateItems.OVERLAP, bool compute_fit = false, bool show_results = false) { var avg_results = new ItemRecommendationEvaluationResults(); Parallel.For(0, (int)split.NumberOfFolds, fold => { try { var split_recommender = (RatingPredictor)recommender.Clone(); // avoid changes in recommender split_recommender.Ratings = split.Train[fold]; split_recommender.Train(); var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[fold]); var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[fold]); IList <int> test_users = test_data_posonly.AllUsers; var fold_results = Items.Evaluate(split_recommender, test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode); if (compute_fit) { fold_results["fit"] = (float)split_recommender.ComputeFit(); } // thread-safe stats lock (avg_results) foreach (var key in fold_results.Keys) { if (avg_results.ContainsKey(key)) { avg_results[key] += fold_results[key]; } else { avg_results[key] = fold_results[key]; } } if (show_results) { Console.Error.WriteLine("fold {0} {1}", fold, fold_results); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); foreach (var key in Items.Measures) { avg_results[key] /= split.NumberOfFolds; } avg_results["num_users"] /= split.NumberOfFolds; avg_results["num_items"] /= split.NumberOfFolds; return(avg_results); }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">an item recommender</param> /// <param name="split">a positive-only feedback dataset split</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> static public void DoRatingBasedRankingIterativeCrossValidation( this RatingPredictor recommender, ISplit <IRatings> split, IList <int> test_users, IList <int> candidate_items, CandidateItems candidate_item_mode, RepeatedEvents repeated_events, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { if (!(recommender is IIterativeModel)) { throw new ArgumentException("recommender must be of type IIterativeModel"); } var split_recommenders = new RatingPredictor[split.NumberOfFolds]; var iterative_recommenders = new IIterativeModel[split.NumberOfFolds]; var fold_results = new ItemRecommendationEvaluationResults[split.NumberOfFolds]; // initial training and evaluation Parallel.For(0, (int)split.NumberOfFolds, i => { try { split_recommenders[i] = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender split_recommenders[i].Ratings = split.Train[i]; split_recommenders[i].Train(); iterative_recommenders[i] = (IIterativeModel)split_recommenders[i]; var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]); var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]); fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events); if (show_fold_results) { Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, iterative_recommenders[i].NumIter); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), iterative_recommenders[0].NumIter); // iterative training and evaluation for (int it = (int)iterative_recommenders[0].NumIter + 1; it <= max_iter; it++) { Parallel.For(0, (int)split.NumberOfFolds, i => { try { iterative_recommenders[i].Iterate(); if (it % find_iter == 0) { var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]); var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]); fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events); if (show_fold_results) { Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, it); } } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), it); } }