/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">an item recommender</param> /// <param name="num_folds">the number of folds</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> static public void DoRatingBasedRankingIterativeCrossValidation( this RatingPredictor recommender, uint num_folds, IList <int> test_users, IList <int> candidate_items, CandidateItems candidate_item_mode, RepeatedEvents repeated_events, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { var split = new RatingCrossValidationSplit(recommender.Ratings, num_folds); recommender.DoRatingBasedRankingIterativeCrossValidation(split, test_users, candidate_items, candidate_item_mode, repeated_events, max_iter, find_iter); }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">an item recommender</param> /// <param name="num_folds">the number of folds</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> static public void DoIterativeCrossValidation( this IRecommender recommender, uint num_folds, IList <int> test_users, IList <int> candidate_items, CandidateItems candidate_item_mode, RepeatedEvents repeated_events, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { if (!(recommender is ItemRecommender)) { throw new ArgumentException("recommender must be of type ItemRecommender"); } var split = new PosOnlyFeedbackCrossValidationSplit <PosOnlyFeedback <SparseBooleanMatrix> >(((ItemRecommender)recommender).Feedback, num_folds); recommender.DoIterativeCrossValidation(split, test_users, candidate_items, candidate_item_mode, repeated_events, max_iter, find_iter); }
/// <summary>Evaluation for rankings of items</summary> /// <remarks> /// User-item combinations that appear in both sets are ignored for the test set, and thus in the evaluation, /// except the boolean argument repeated_events is set. /// /// The evaluation measures are listed in the Measures property. /// Additionally, 'num_users' and 'num_items' report the number of users that were used to compute the results /// and the number of items that were taken into account. /// /// Literature: /// <list type="bullet"> /// <item><description> /// C. Manning, P. Raghavan, H. Schütze: Introduction to Information Retrieval, Cambridge University Press, 2008 /// </description></item> /// </list> /// /// On multi-core/multi-processor systems, the routine tries to use as many cores as possible, /// which should to an almost linear speed-up. /// </remarks> /// <param name="recommender">item recommender</param> /// <param name="test">test cases</param> /// <param name="training">training data</param> /// <param name="test_users">a list of integers with all test users; if null, use all users in the test cases</param> /// <param name="candidate_items">a list of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="n">length of the item list to evaluate -- if set to -1 (default), use the complete list, otherwise compute evaluation measures on the top n items</param> /// <returns>a dictionary containing the evaluation results (default is false)</returns> public static ItemRecommendationEvaluationResults Evaluate( this IRecommender recommender, IPosOnlyFeedback test, IPosOnlyFeedback training, IList<int> test_users = null, IList<int> candidate_items = null, CandidateItems candidate_item_mode = CandidateItems.OVERLAP, RepeatedEvents repeated_events = RepeatedEvents.No, int n = -1) { if (test_users == null) test_users = test.AllUsers; candidate_items = Candidates(candidate_items, candidate_item_mode, test, training); var result = new ItemRecommendationEvaluationResults(); // make sure that the user matrix is completely initialized before entering parallel code var training_user_matrix = training.UserMatrix; var test_user_matrix = test.UserMatrix; int num_users = 0; Parallel.ForEach(test_users, user_id => { try { var correct_items = new HashSet<int>(test_user_matrix[user_id]); correct_items.IntersectWith(candidate_items); if (correct_items.Count == 0) return; var ignore_items_for_this_user = new HashSet<int>( repeated_events == RepeatedEvents.Yes || training_user_matrix[user_id] == null ? new int[0] : training_user_matrix[user_id] ); ignore_items_for_this_user.IntersectWith(candidate_items); int num_candidates_for_this_user = candidate_items.Count - ignore_items_for_this_user.Count; if (correct_items.Count == num_candidates_for_this_user) return; var prediction = recommender.Recommend(user_id, candidate_items:candidate_items, n:n, ignore_items:ignore_items_for_this_user); var prediction_list = (from t in prediction select t.Item1).ToArray(); int num_dropped_items = num_candidates_for_this_user - prediction.Count; double auc = AUC.Compute(prediction_list, correct_items, num_dropped_items); double map = PrecisionAndRecall.AP(prediction_list, correct_items); double ndcg = NDCG.Compute(prediction_list, correct_items); double rr = ReciprocalRank.Compute(prediction_list, correct_items); var positions = new int[] { 5, 10 }; var prec = PrecisionAndRecall.PrecisionAt(prediction_list, correct_items, positions); var recall = PrecisionAndRecall.RecallAt(prediction_list, correct_items, positions); // thread-safe incrementing lock (result) { num_users++; result["AUC"] += (float) auc; result["MAP"] += (float) map; result["NDCG"] += (float) ndcg; result["MRR"] += (float) rr; result["prec@5"] += (float) prec[5]; result["prec@10"] += (float) prec[10]; result["recall@5"] += (float) recall[5]; result["recall@10"] += (float) recall[10]; } if (num_users % 1000 == 0) Console.Error.Write("."); if (num_users % 60000 == 0) Console.Error.WriteLine(); } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); foreach (string measure in Measures) result[measure] /= num_users; result["num_users"] = num_users; result["num_lists"] = num_users; result["num_items"] = candidate_items.Count; return result; }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">an item recommender</param> /// <param name="split">a positive-only feedback dataset split</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> public static void DoIterativeCrossValidation( this IRecommender recommender, ISplit<IPosOnlyFeedback> split, IList<int> test_users, IList<int> candidate_items, CandidateItems candidate_item_mode, RepeatedEvents repeated_events, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { if (!(recommender is IIterativeModel)) throw new ArgumentException("recommender must be of type IIterativeModel"); if (!(recommender is ItemRecommender)) throw new ArgumentException("recommender must be of type ItemRecommender"); var split_recommenders = new ItemRecommender[split.NumberOfFolds]; var iterative_recommenders = new IIterativeModel[split.NumberOfFolds]; var fold_results = new ItemRecommendationEvaluationResults[split.NumberOfFolds]; // initial training and evaluation Parallel.For(0, (int) split.NumberOfFolds, i => { try { split_recommenders[i] = (ItemRecommender) recommender.Clone(); // to avoid changes in recommender split_recommenders[i].Feedback = split.Train[i]; split_recommenders[i].Train(); iterative_recommenders[i] = (IIterativeModel) split_recommenders[i]; fold_results[i] = Items.Evaluate(split_recommenders[i], split.Test[i], split.Train[i], test_users, candidate_items, candidate_item_mode, repeated_events); if (show_fold_results) Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, iterative_recommenders[i].NumIter); } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), iterative_recommenders[0].NumIter); // iterative training and evaluation for (int it = (int) iterative_recommenders[0].NumIter + 1; it <= max_iter; it++) { Parallel.For(0, (int) split.NumberOfFolds, i => { try { iterative_recommenders[i].Iterate(); if (it % find_iter == 0) { fold_results[i] = Items.Evaluate(split_recommenders[i], split.Test[i], split.Train[i], test_users, candidate_items, candidate_item_mode, repeated_events); if (show_fold_results) Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, it); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), it); } }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">an item recommender</param> /// <param name="num_folds">the number of folds</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> public static void DoIterativeCrossValidation( this IRecommender recommender, uint num_folds, IList<int> test_users, IList<int> candidate_items, CandidateItems candidate_item_mode, RepeatedEvents repeated_events, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { if (!(recommender is ItemRecommender)) throw new ArgumentException("recommender must be of type ItemRecommender"); var split = new PosOnlyFeedbackCrossValidationSplit<PosOnlyFeedback<SparseBooleanMatrix>>(((ItemRecommender) recommender).Feedback, num_folds); recommender.DoIterativeCrossValidation(split, test_users, candidate_items, candidate_item_mode, repeated_events, max_iter, find_iter); }
/// <summary>Evaluation for rankings of items</summary> /// <remarks> /// User-item combinations that appear in both sets are ignored for the test set, and thus in the evaluation, /// except the boolean argument repeated_events is set. /// /// The evaluation measures are listed in the Measures property. /// Additionally, 'num_users' and 'num_items' report the number of users that were used to compute the results /// and the number of items that were taken into account. /// /// Literature: /// <list type="bullet"> /// <item><description> /// C. Manning, P. Raghavan, H. Schütze: Introduction to Information Retrieval, Cambridge University Press, 2008 /// </description></item> /// </list> /// /// On multi-core/multi-processor systems, the routine tries to use as many cores as possible, /// which should to an almost linear speed-up. /// </remarks> /// <param name="recommender">item recommender</param> /// <param name="test">test cases</param> /// <param name="training">training data</param> /// <param name="test_users">a list of integers with all test users; if null, use all users in the test cases</param> /// <param name="candidate_items">a list of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="n">length of the item list to evaluate -- if set to -1 (default), use the complete list, otherwise compute evaluation measures on the top n items</param> /// <returns>a dictionary containing the evaluation results (default is false)</returns> static public ItemRecommendationEvaluationResults Evaluate( this IRecommender recommender, IPosOnlyFeedback test, IPosOnlyFeedback training, IList <int> test_users = null, IList <int> candidate_items = null, CandidateItems candidate_item_mode = CandidateItems.OVERLAP, RepeatedEvents repeated_events = RepeatedEvents.No, int n = -1) { if (test_users == null) { test_users = test.AllUsers; } candidate_items = Candidates(candidate_items, candidate_item_mode, test, training); var result = new ItemRecommendationEvaluationResults(); // make sure that the user matrix is completely initialized before entering parallel code var training_user_matrix = training.UserMatrix; var test_user_matrix = test.UserMatrix; int num_users = 0; Parallel.ForEach(test_users, user_id => { try { var correct_items = new HashSet <int>(test_user_matrix[user_id]); correct_items.IntersectWith(candidate_items); if (correct_items.Count == 0) { return; } var ignore_items_for_this_user = new HashSet <int>( repeated_events == RepeatedEvents.Yes || training_user_matrix[user_id] == null ? new int[0] : training_user_matrix[user_id] ); ignore_items_for_this_user.IntersectWith(candidate_items); int num_candidates_for_this_user = candidate_items.Count - ignore_items_for_this_user.Count; if (correct_items.Count == num_candidates_for_this_user) { return; } var prediction = recommender.Recommend(user_id, candidate_items: candidate_items, n: n, ignore_items: ignore_items_for_this_user); var prediction_list = (from t in prediction select t.Item1).ToArray(); int num_dropped_items = num_candidates_for_this_user - prediction.Count; double auc = AUC.Compute(prediction_list, correct_items, num_dropped_items); double map = PrecisionAndRecall.AP(prediction_list, correct_items); double ndcg = NDCG.Compute(prediction_list, correct_items); double rr = ReciprocalRank.Compute(prediction_list, correct_items); var positions = new int[] { 5, 10 }; var prec = PrecisionAndRecall.PrecisionAt(prediction_list, correct_items, positions); var recall = PrecisionAndRecall.RecallAt(prediction_list, correct_items, positions); // thread-safe incrementing lock (result) { num_users++; result["AUC"] += (float)auc; result["MAP"] += (float)map; result["NDCG"] += (float)ndcg; result["MRR"] += (float)rr; result["prec@5"] += (float)prec[5]; result["prec@10"] += (float)prec[10]; result["recall@5"] += (float)recall[5]; result["recall@10"] += (float)recall[10]; } if (num_users % 1000 == 0) { Console.Error.Write("."); } if (num_users % 60000 == 0) { Console.Error.WriteLine(); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); foreach (string measure in Measures) { result[measure] /= num_users; } result["num_users"] = num_users; result["num_lists"] = num_users; result["num_items"] = candidate_items.Count; return(result); }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">an item recommender</param> /// <param name="split">a positive-only feedback dataset split</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> static public void DoRatingBasedRankingIterativeCrossValidation( this RatingPredictor recommender, ISplit <IRatings> split, IList <int> test_users, IList <int> candidate_items, CandidateItems candidate_item_mode, RepeatedEvents repeated_events, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { if (!(recommender is IIterativeModel)) { throw new ArgumentException("recommender must be of type IIterativeModel"); } var split_recommenders = new RatingPredictor[split.NumberOfFolds]; var iterative_recommenders = new IIterativeModel[split.NumberOfFolds]; var fold_results = new ItemRecommendationEvaluationResults[split.NumberOfFolds]; // initial training and evaluation Parallel.For(0, (int)split.NumberOfFolds, i => { try { split_recommenders[i] = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender split_recommenders[i].Ratings = split.Train[i]; split_recommenders[i].Train(); iterative_recommenders[i] = (IIterativeModel)split_recommenders[i]; var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]); var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]); fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events); if (show_fold_results) { Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, iterative_recommenders[i].NumIter); } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), iterative_recommenders[0].NumIter); // iterative training and evaluation for (int it = (int)iterative_recommenders[0].NumIter + 1; it <= max_iter; it++) { Parallel.For(0, (int)split.NumberOfFolds, i => { try { iterative_recommenders[i].Iterate(); if (it % find_iter == 0) { var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]); var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]); fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events); if (show_fold_results) { Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, it); } } } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), it); } }
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary> /// <param name="recommender">an item recommender</param> /// <param name="num_folds">the number of folds</param> /// <param name="test_users">a collection of integers with all test users</param> /// <param name="candidate_items">a collection of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="max_iter">the maximum number of iterations</param> /// <param name="find_iter">the report interval</param> /// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param> static public void DoRatingBasedRankingIterativeCrossValidation( this RatingPredictor recommender, uint num_folds, IList<int> test_users, IList<int> candidate_items, CandidateItems candidate_item_mode, RepeatedEvents repeated_events, uint max_iter, uint find_iter = 1, bool show_fold_results = false) { var split = new RatingCrossValidationSplit(recommender.Ratings, num_folds); recommender.DoRatingBasedRankingIterativeCrossValidation(split, test_users, candidate_items, candidate_item_mode, repeated_events, max_iter, find_iter); }
/// <summary>Evaluation for rankings of items</summary> /// <remarks> /// User-item combinations that appear in both sets are ignored for the test set, and thus in the evaluation, /// except the boolean argument repeated_events is set. /// /// The evaluation measures are listed in the Measures property. /// Additionally, 'num_users' and 'num_items' report the number of users that were used to compute the results /// and the number of items that were taken into account. /// /// Literature: /// <list type="bullet"> /// <item><description> /// C. Manning, P. Raghavan, H. Schütze: Introduction to Information Retrieval, Cambridge University Press, 2008 /// </description></item> /// </list> /// /// On multi-core/multi-processor systems, the routine tries to use as many cores as possible, /// which should to an almost linear speed-up. /// </remarks> /// <param name="recommender">item recommender</param> /// <param name="test">test cases</param> /// <param name="training">training data</param> /// <param name="test_users">a list of integers with all test users; if null, use all users in the test cases</param> /// <param name="candidate_items">a list of integers with all candidate items</param> /// <param name="candidate_item_mode">the mode used to determine the candidate items</param> /// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param> /// <param name="n">length of the item list to evaluate -- if set to -1 (default), use the complete list, otherwise compute evaluation measures on the top n items</param> /// <returns>a dictionary containing the evaluation results (default is false)</returns> public static ItemRecommendationEvaluationResults Evaluate( this IRecommender recommender, IPosOnlyFeedback test, IPosOnlyFeedback training, IList<int> test_users = null, IList<int> candidate_items = null, CandidateItems candidate_item_mode = CandidateItems.OVERLAP, RepeatedEvents repeated_events = RepeatedEvents.No, int n = -1) { switch (candidate_item_mode) { case CandidateItems.TRAINING: candidate_items = training.AllItems; break; case CandidateItems.TEST: candidate_items = test.AllItems; break; case CandidateItems.OVERLAP: candidate_items = new List<int>(test.AllItems.Intersect(training.AllItems)); break; case CandidateItems.UNION: candidate_items = new List<int>(test.AllItems.Union(training.AllItems)); break; } if (candidate_items == null) throw new ArgumentNullException("candidate_items"); if (test_users == null) test_users = test.AllUsers; int num_users = 0; var result = new ItemRecommendationEvaluationResults(); // make sure that the user matrix is completely initialized before entering parallel code var training_user_matrix = training.UserMatrix; var test_user_matrix = test.UserMatrix; Parallel.ForEach(test_users, user_id => { try { var correct_items = new HashSet<int>(test_user_matrix[user_id]); correct_items.IntersectWith(candidate_items); // the number of items that will be used for this user var candidate_items_in_train = new HashSet<int>(training_user_matrix[user_id]); candidate_items_in_train.IntersectWith(candidate_items); int num_eval_items = candidate_items.Count - (repeated_events == RepeatedEvents.Yes ? 0 : candidate_items_in_train.Count()); // skip all users that have 0 or #candidate_items test items if (correct_items.Count == 0) return; if (num_eval_items == correct_items.Count) return; ICollection<int> ignore_items = repeated_events == RepeatedEvents.Yes ? new int[0] : training_user_matrix[user_id]; var prediction = recommender.Recommend(user_id, candidate_items:candidate_items, n:n, ignore_items:ignore_items); var prediction_list = (from t in prediction select t.Item1).ToArray(); double auc = AUC.Compute(prediction_list, correct_items, ignore_items); double map = PrecisionAndRecall.AP(prediction_list, correct_items, ignore_items); double ndcg = NDCG.Compute(prediction_list, correct_items, ignore_items); double rr = ReciprocalRank.Compute(prediction_list, correct_items, ignore_items); var positions = new int[] { 5, 10 }; var prec = PrecisionAndRecall.PrecisionAt(prediction_list, correct_items, ignore_items, positions); var recall = PrecisionAndRecall.RecallAt(prediction_list, correct_items, ignore_items, positions); // thread-safe incrementing lock (result) { num_users++; result["AUC"] += (float) auc; result["MAP"] += (float) map; result["NDCG"] += (float) ndcg; result["MRR"] += (float) rr; result["prec@5"] += (float) prec[5]; result["prec@10"] += (float) prec[10]; result["recall@5"] += (float) recall[5]; result["recall@10"] += (float) recall[10]; } if (num_users % 1000 == 0) Console.Error.Write("."); if (num_users % 60000 == 0) Console.Error.WriteLine(); } catch (Exception e) { Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace); throw; } }); foreach (string measure in Measures) result[measure] /= num_users; result["num_users"] = num_users; result["num_lists"] = num_users; result["num_items"] = candidate_items.Count; return result; }