/// <param name="candidate_items">a list of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="test">test cases</param>
/// <param name="training">training data</param>
static public IList <int> Candidates(
IList <int> candidate_items,
CandidateItems candidate_item_mode,
IPosOnlyFeedback test,
IPosOnlyFeedback training)
{
switch (candidate_item_mode)
{
case CandidateItems.TRAINING: return(training.AllItems);
case CandidateItems.TEST: return(test.AllItems);
case CandidateItems.OVERLAP:
var result = test.AllItems.Intersect(training.AllItems).ToList();
result.Shuffle();
return(result);
case CandidateItems.UNION:
result = test.AllItems.Union(training.AllItems).ToList();
result.Shuffle();
return(result);
case CandidateItems.EXPLICIT:
if (candidate_items == null)
{
throw new ArgumentNullException("candidate_items");
}
return(candidate_items);
default:
throw new ArgumentException("Unknown candidate_item_mode: " + candidate_item_mode.ToString());
}
}
protected override void LoadData()
{
base.LoadData();
// test users
if (test_users_file != null)
test_users = user_mapping.ToInternalID( File.ReadLines(Path.Combine(data_dir, test_users_file)).ToArray() );
else
test_users = test_data != null ? test_data.AllUsers : training_data.AllUsers;
// candidate items
if (candidate_items_file != null)
candidate_items = item_mapping.ToInternalID( File.ReadLines(Path.Combine(data_dir, candidate_items_file)).ToArray() );
else if (all_items)
candidate_items = Enumerable.Range(0, item_mapping.InternalIDs.Max() + 1).ToArray();
if (candidate_items != null)
eval_item_mode = CandidateItems.EXPLICIT;
else if (in_training_items)
eval_item_mode = CandidateItems.TRAINING;
else if (in_test_items)
eval_item_mode = CandidateItems.TEST;
else if (overlap_items)
eval_item_mode = CandidateItems.OVERLAP;
else
eval_item_mode = CandidateItems.UNION;
}
/// <param name="candidate_items">a list of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="test">test cases</param>
/// <param name="training">training data</param>
public static IList<int> Candidates(
IList<int> candidate_items,
CandidateItems candidate_item_mode,
IPosOnlyFeedback test,
IPosOnlyFeedback training)
{
switch (candidate_item_mode)
{
case CandidateItems.TRAINING: return training.AllItems;
case CandidateItems.TEST: return test.AllItems;
case CandidateItems.OVERLAP:
var result = test.AllItems.Intersect(training.AllItems).ToList();
result.Shuffle();
return result;
case CandidateItems.UNION:
result = test.AllItems.Union(training.AllItems).ToList();
result.Shuffle();
return result;
case CandidateItems.EXPLICIT:
if (candidate_items == null)
throw new ArgumentNullException("candidate_items");
return candidate_items;
default:
throw new ArgumentException("Unknown candidate_item_mode: " + candidate_item_mode.ToString());
}
}
/// <param name="candidate_items">a list of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="test">test cases</param>
/// <param name="training">training data</param>
static public IList<int> Candidates(
IList<int> candidate_items,
CandidateItems candidate_item_mode,
IPosOnlyFeedback test,
IPosOnlyFeedback training)
{
IList<int> test_items = (test != null) ? test.AllItems : new int[0];
IList<int> result = null;
switch (candidate_item_mode)
{
case CandidateItems.TRAINING:
result = training.AllItems.ToArray();
break;
case CandidateItems.TEST:
result = test.AllItems.ToArray();
break;
case CandidateItems.OVERLAP:
result = test_items.Intersect(training.AllItems).ToList();
break;
case CandidateItems.UNION:
result = test_items.Union(training.AllItems).ToList();
break;
case CandidateItems.EXPLICIT:
if (candidate_items == null)
throw new ArgumentNullException("candidate_items");
result = candidate_items.ToArray();
break;
default:
throw new ArgumentException("Unknown candidate_item_mode: " + candidate_item_mode.ToString());
}
result.Shuffle();
return result;
}
/// <summary>Computes the AUC fit of a recommender on the training data</summary>
/// <returns>the AUC on the training data</returns>
/// <param name='recommender'>the item recommender to evaluate</param>
/// <param name="test_users">a list of integers with all test users; if null, use all users in the test cases</param>
/// <param name="candidate_items">a list of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
public static double ComputeFit(
this ItemRecommender recommender,
IList<int> test_users = null,
IList<int> candidate_items = null,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP)
{
return recommender.Evaluate(recommender.Feedback, recommender.Feedback, test_users, candidate_items, candidate_item_mode, true)["RMSE"];
}
/// <summary>Online evaluation for rankings of items</summary>
/// <remarks>
/// The evaluation protocol works as follows:
/// For every test user, evaluate on the test items, and then add the those test items to the training set and perform an incremental update.
/// The sequence of users is random.
/// </remarks>
/// <param name="recommender">the item recommender to be evaluated</param>
/// <param name="test">test cases</param>
/// <param name="training">training data (must be connected to the recommender's training data)</param>
/// <param name="test_users">a list of all test user IDs</param>
/// <param name="candidate_items">a list of all candidate item IDs</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <returns>a dictionary containing the evaluation results (averaged by user)</returns>
public static ItemRecommendationEvaluationResults EvaluateOnline(
this IRecommender recommender,
IPosOnlyFeedback test, IPosOnlyFeedback training,
IList<int> test_users, IList<int> candidate_items,
CandidateItems candidate_item_mode)
{
var incremental_recommender = recommender as IIncrementalItemRecommender;
if (incremental_recommender == null)
throw new ArgumentException("recommender must be of type IIncrementalItemRecommender");
// prepare candidate items once to avoid recreating them
switch (candidate_item_mode)
{
case CandidateItems.TRAINING: candidate_items = training.AllItems; break;
case CandidateItems.TEST: candidate_items = test.AllItems; break;
case CandidateItems.OVERLAP: candidate_items = new List<int>(test.AllItems.Intersect(training.AllItems)); break;
case CandidateItems.UNION: candidate_items = new List<int>(test.AllItems.Union(training.AllItems)); break;
}
test_users.Shuffle();
var results_by_user = new Dictionary<int, ItemRecommendationEvaluationResults>();
foreach (int user_id in test_users)
{
if (candidate_items.Intersect(test.ByUser[user_id]).Count() == 0)
continue;
// prepare data
var current_test_data = new PosOnlyFeedback<SparseBooleanMatrix>();
foreach (int index in test.ByUser[user_id])
current_test_data.Add(user_id, test.Items[index]);
// evaluate user
var current_result = Items.Evaluate(recommender, current_test_data, training, current_test_data.AllUsers, candidate_items, CandidateItems.EXPLICIT);
results_by_user[user_id] = current_result;
// update recommender
var tuples = new List<Tuple<int, int>>();
foreach (int index in test.ByUser[user_id])
tuples.Add(Tuple.Create(user_id, test.Items[index]));
incremental_recommender.AddFeedback(tuples);
}
var results = new ItemRecommendationEvaluationResults();
foreach (int u in results_by_user.Keys)
foreach (string measure in Items.Measures)
results[measure] += results_by_user[u][measure];
foreach (string measure in Items.Measures)
results[measure] /= results_by_user.Count;
results["num_users"] = results_by_user.Count;
results["num_items"] = candidate_items.Count;
results["num_lists"] = results_by_user.Count;
return results;
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="num_folds">the number of folds</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public EvaluationResults DoRatingBasedRankingCrossValidation(
this RatingPredictor recommender,
uint num_folds,
IList<int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var split = new RatingCrossValidationSplit(recommender.Ratings, num_folds);
return recommender.DoRatingBasedRankingCrossValidation(split, candidate_items, candidate_item_mode, compute_fit, show_results);
}
/// <summary>Computes the AUC fit of a recommender on the training data</summary>
/// <returns>the AUC on the training data</returns>
/// <param name='recommender'>the item recommender to evaluate</param>
/// <param name="test_users">a list of integers with all test users; if null, use all users in the test cases</param>
/// <param name="candidate_items">a list of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
public static double ComputeFit(
this ItemRecommender recommender,
IList <int> test_users = null,
IList <int> candidate_items = null,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP)
{
return(recommender.Evaluate(
recommender.Feedback, recommender.Feedback,
test_users, candidate_items,
candidate_item_mode, RepeatedEvents.Yes)["AUC"]);
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="num_folds">the number of folds</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public EvaluationResults DoRatingBasedRankingCrossValidation(
this RatingPredictor recommender,
uint num_folds,
IList <int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var split = new RatingCrossValidationSplit(recommender.Ratings, num_folds);
return(recommender.DoRatingBasedRankingCrossValidation(split, candidate_items, candidate_item_mode, compute_fit, show_results));
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a dataset split</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
public static ItemRecommendationEvaluationResults DoCrossValidation(
this IRecommender recommender,
ISplit<IPosOnlyFeedback> split,
IList<int> test_users,
IList<int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var avg_results = new ItemRecommendationEvaluationResults();
if (!(recommender is ItemRecommender))
throw new ArgumentException("recommender must be of type ItemRecommender");
Parallel.For(0, (int) split.NumberOfFolds, fold =>
{
try
{
var split_recommender = (ItemRecommender) recommender.Clone(); // avoid changes in recommender
split_recommender.Feedback = split.Train[fold];
split_recommender.Train();
var fold_results = Items.Evaluate(split_recommender, split.Test[fold], split.Train[fold], test_users, candidate_items, candidate_item_mode);
if (compute_fit)
fold_results["fit"] = (float) split_recommender.ComputeFit();
// thread-safe stats
lock (avg_results)
foreach (var key in fold_results.Keys)
if (avg_results.ContainsKey(key))
avg_results[key] += fold_results[key];
else
avg_results[key] = fold_results[key];
if (show_results)
Console.Error.WriteLine("fold {0} {1}", fold, fold_results);
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (var key in Items.Measures)
avg_results[key] /= split.NumberOfFolds;
avg_results["num_users"] /= split.NumberOfFolds;
avg_results["num_items"] /= split.NumberOfFolds;
if (compute_fit)
avg_results["fit"] /= split.NumberOfFolds;
return avg_results;
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a dataset split</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public EvaluationResults DoRatingBasedRankingCrossValidation(
this RatingPredictor recommender,
ISplit<IRatings> split,
IList<int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var avg_results = new ItemRecommendationEvaluationResults();
Parallel.For(0, (int) split.NumberOfFolds, fold =>
{
try
{
var split_recommender = (RatingPredictor) recommender.Clone(); // avoid changes in recommender
split_recommender.Ratings = split.Train[fold];
split_recommender.Train();
var test_data_posonly = new PosOnlyFeedback<SparseBooleanMatrix>(split.Test[fold]);
var training_data_posonly = new PosOnlyFeedback<SparseBooleanMatrix>(split.Train[fold]);
IList<int> test_users = test_data_posonly.AllUsers;
var fold_results = Items.Evaluate(split_recommender, test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode);
if (compute_fit)
fold_results["fit"] = (float) split_recommender.ComputeFit();
// thread-safe stats
lock (avg_results)
foreach (var key in fold_results.Keys)
if (avg_results.ContainsKey(key))
avg_results[key] += fold_results[key];
else
avg_results[key] = fold_results[key];
if (show_results)
Console.Error.WriteLine("fold {0} {1}", fold, fold_results);
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (var key in Items.Measures)
avg_results[key] /= split.NumberOfFolds;
avg_results["num_users"] /= split.NumberOfFolds;
avg_results["num_items"] /= split.NumberOfFolds;
return avg_results;
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="num_folds">the number of folds</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
static public void DoRatingBasedRankingIterativeCrossValidation(
this RatingPredictor recommender,
uint num_folds,
IList <int> test_users,
IList <int> candidate_items,
CandidateItems candidate_item_mode,
RepeatedEvents repeated_events,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
var split = new RatingCrossValidationSplit(recommender.Ratings, num_folds);
recommender.DoRatingBasedRankingIterativeCrossValidation(split, test_users, candidate_items, candidate_item_mode, repeated_events, max_iter, find_iter);
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="num_folds">the number of folds</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
public static ItemRecommendationEvaluationResults DoCrossValidation(
this IRecommender recommender,
uint num_folds,
IList<int> test_users,
IList<int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
if (!(recommender is ItemRecommender))
throw new ArgumentException("recommender must be of type ItemRecommender");
var split = new PosOnlyFeedbackCrossValidationSplit<PosOnlyFeedback<SparseBooleanMatrix>>(((ItemRecommender) recommender).Feedback, num_folds);
return recommender.DoCrossValidation(split, test_users, candidate_items, candidate_item_mode, compute_fit, show_results);
}
protected override void LoadData()
{
base.LoadData();
// test users
if (test_users_file != null)
{
test_users = user_mapping.ToInternalID(File.ReadLines(Path.Combine(data_dir, test_users_file)).ToArray());
}
else
{
test_users = test_data != null ? test_data.AllUsers : training_data.AllUsers;
}
// candidate items
if (candidate_items_file != null)
{
candidate_items = item_mapping.ToInternalID(File.ReadLines(Path.Combine(data_dir, candidate_items_file)).ToArray());
}
else if (all_items)
{
candidate_items = Enumerable.Range(0, item_mapping.InternalIDs.Max() + 1).ToArray();
}
if (candidate_items != null)
{
eval_item_mode = CandidateItems.EXPLICIT;
}
else if (in_training_items)
{
eval_item_mode = CandidateItems.TRAINING;
}
else if (in_test_items)
{
eval_item_mode = CandidateItems.TEST;
}
else if (overlap_items)
{
eval_item_mode = CandidateItems.OVERLAP;
}
else
{
eval_item_mode = CandidateItems.UNION;
}
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="num_folds">the number of folds</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public ItemRecommendationEvaluationResults DoCrossValidation(
this IRecommender recommender,
uint num_folds,
IList <int> test_users,
IList <int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
if (!(recommender is ItemRecommender))
{
throw new ArgumentException("recommender must be of type ItemRecommender");
}
var split = new PosOnlyFeedbackCrossValidationSplit <PosOnlyFeedback <SparseBooleanMatrix> >(((ItemRecommender)recommender).Feedback, num_folds);
return(recommender.DoCrossValidation(split, test_users, candidate_items, candidate_item_mode, compute_fit, show_results));
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="num_folds">the number of folds</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
static public void DoIterativeCrossValidation(
this IRecommender recommender,
uint num_folds,
IList <int> test_users,
IList <int> candidate_items,
CandidateItems candidate_item_mode,
RepeatedEvents repeated_events,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
if (!(recommender is ItemRecommender))
{
throw new ArgumentException("recommender must be of type ItemRecommender");
}
var split = new PosOnlyFeedbackCrossValidationSplit <PosOnlyFeedback <SparseBooleanMatrix> >(((ItemRecommender)recommender).Feedback, num_folds);
recommender.DoIterativeCrossValidation(split, test_users, candidate_items, candidate_item_mode, repeated_events, max_iter, find_iter);
}
/// <summary>Evaluation for rankings of items</summary>
/// <remarks>
/// User-item combinations that appear in both sets are ignored for the test set, and thus in the evaluation,
/// except the boolean argument repeated_events is set.
///
/// The evaluation measures are listed in the Measures property.
/// Additionally, 'num_users' and 'num_items' report the number of users that were used to compute the results
/// and the number of items that were taken into account.
///
/// Literature:
/// <list type="bullet">
/// <item><description>
/// C. Manning, P. Raghavan, H. Schütze: Introduction to Information Retrieval, Cambridge University Press, 2008
/// </description></item>
/// </list>
///
/// On multi-core/multi-processor systems, the routine tries to use as many cores as possible,
/// which should to an almost linear speed-up.
/// </remarks>
/// <param name="recommender">item recommender</param>
/// <param name="test">test cases</param>
/// <param name="training">training data</param>
/// <param name="test_users">a list of integers with all test users; if null, use all users in the test cases</param>
/// <param name="candidate_items">a list of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="n">length of the item list to evaluate -- if set to -1 (default), use the complete list, otherwise compute evaluation measures on the top n items</param>
/// <returns>a dictionary containing the evaluation results (default is false)</returns>
static public ItemRecommendationEvaluationResults Evaluate(
this IRecommender recommender,
IPosOnlyFeedback test,
IPosOnlyFeedback training,
IList <int> test_users = null,
IList <int> candidate_items = null,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
RepeatedEvents repeated_events = RepeatedEvents.No,
int n = -1)
{
if (test_users == null)
{
test_users = test.AllUsers;
}
candidate_items = Candidates(candidate_items, candidate_item_mode, test, training);
var result = new ItemRecommendationEvaluationResults();
// make sure that the user matrix is completely initialized before entering parallel code
var training_user_matrix = training.UserMatrix;
var test_user_matrix = test.UserMatrix;
int num_users = 0;
Parallel.ForEach(test_users, user_id => {
try
{
var correct_items = new HashSet <int>(test_user_matrix[user_id]);
correct_items.IntersectWith(candidate_items);
if (correct_items.Count == 0)
{
return;
}
var ignore_items_for_this_user = new HashSet <int>(
repeated_events == RepeatedEvents.Yes || training_user_matrix[user_id] == null ? new int[0] : training_user_matrix[user_id]
);
ignore_items_for_this_user.IntersectWith(candidate_items);
int num_candidates_for_this_user = candidate_items.Count - ignore_items_for_this_user.Count;
if (correct_items.Count == num_candidates_for_this_user)
{
return;
}
var prediction = recommender.Recommend(user_id, candidate_items: candidate_items, n: n, ignore_items: ignore_items_for_this_user);
var prediction_list = (from t in prediction select t.Item1).ToArray();
int num_dropped_items = num_candidates_for_this_user - prediction.Count;
double auc = AUC.Compute(prediction_list, correct_items, num_dropped_items);
double map = PrecisionAndRecall.AP(prediction_list, correct_items);
double ndcg = NDCG.Compute(prediction_list, correct_items);
double rr = ReciprocalRank.Compute(prediction_list, correct_items);
var positions = new int[] { 5, 10 };
var prec = PrecisionAndRecall.PrecisionAt(prediction_list, correct_items, positions);
var recall = PrecisionAndRecall.RecallAt(prediction_list, correct_items, positions);
// thread-safe incrementing
lock (result)
{
num_users++;
result["AUC"] += (float)auc;
result["MAP"] += (float)map;
result["NDCG"] += (float)ndcg;
result["MRR"] += (float)rr;
result["prec@5"] += (float)prec[5];
result["prec@10"] += (float)prec[10];
result["recall@5"] += (float)recall[5];
result["recall@10"] += (float)recall[10];
}
if (num_users % 1000 == 0)
{
Console.Error.Write(".");
}
if (num_users % 60000 == 0)
{
Console.Error.WriteLine();
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (string measure in Measures)
{
result[measure] /= num_users;
}
result["num_users"] = num_users;
result["num_lists"] = num_users;
result["num_items"] = candidate_items.Count;
return(result);
}
/// <summary>Online evaluation for rankings of items</summary>
/// <remarks>
/// The evaluation protocol works as follows:
/// For every test user, evaluate on the test items, and then add the those test items to the training set and perform an incremental update.
/// The sequence of users is random.
/// </remarks>
/// <param name="recommender">the item recommender to be evaluated</param>
/// <param name="test">test cases</param>
/// <param name="training">training data (must be connected to the recommender's training data)</param>
/// <param name="test_users">a list of all test user IDs</param>
/// <param name="candidate_items">a list of all candidate item IDs</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <returns>a dictionary containing the evaluation results (averaged by user)</returns>
static public ItemRecommendationEvaluationResults EvaluateOnline(
this IRecommender recommender,
IPosOnlyFeedback test, IPosOnlyFeedback training,
IList <int> test_users, IList <int> candidate_items,
CandidateItems candidate_item_mode)
{
var incremental_recommender = recommender as IIncrementalItemRecommender;
if (incremental_recommender == null)
{
throw new ArgumentException("recommender must be of type IIncrementalItemRecommender");
}
candidate_items = Items.Candidates(candidate_items, candidate_item_mode, test, training);
test_users.Shuffle();
var results_by_user = new Dictionary <int, ItemRecommendationEvaluationResults>();
foreach (int user_id in test_users)
{
if (candidate_items.Intersect(test.ByUser[user_id]).Count() == 0)
{
continue;
}
// prepare data
var current_test_data = new PosOnlyFeedback <SparseBooleanMatrix>();
foreach (int index in test.ByUser[user_id])
{
current_test_data.Add(user_id, test.Items[index]);
}
// evaluate user
var current_result = Items.Evaluate(recommender, current_test_data, training, current_test_data.AllUsers, candidate_items, CandidateItems.EXPLICIT);
results_by_user[user_id] = current_result;
// update recommender
var tuples = new List <Tuple <int, int> >();
foreach (int index in test.ByUser[user_id])
{
tuples.Add(Tuple.Create(user_id, test.Items[index]));
}
incremental_recommender.AddFeedback(tuples);
// TODO candidate_items should be updated properly
}
var results = new ItemRecommendationEvaluationResults();
foreach (int u in results_by_user.Keys)
{
foreach (string measure in Items.Measures)
{
results[measure] += results_by_user[u][measure];
}
}
foreach (string measure in Items.Measures)
{
results[measure] /= results_by_user.Count;
}
results["num_users"] = results_by_user.Count;
results["num_items"] = candidate_items.Count;
results["num_lists"] = results_by_user.Count;
return(results);
}
/// <summary>Evaluation for rankings of items</summary>
/// <remarks>
/// User-item combinations that appear in both sets are ignored for the test set, and thus in the evaluation,
/// except the boolean argument repeated_events is set.
///
/// The evaluation measures are listed in the Measures property.
/// Additionally, 'num_users' and 'num_items' report the number of users that were used to compute the results
/// and the number of items that were taken into account.
///
/// Literature:
/// <list type="bullet">
/// <item><description>
/// C. Manning, P. Raghavan, H. Schütze: Introduction to Information Retrieval, Cambridge University Press, 2008
/// </description></item>
/// </list>
///
/// On multi-core/multi-processor systems, the routine tries to use as many cores as possible,
/// which should to an almost linear speed-up.
/// </remarks>
/// <param name="recommender">item recommender</param>
/// <param name="test">test cases</param>
/// <param name="training">training data</param>
/// <param name="test_users">a list of integers with all test users; if null, use all users in the test cases</param>
/// <param name="candidate_items">a list of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="n">length of the item list to evaluate -- if set to -1 (default), use the complete list, otherwise compute evaluation measures on the top n items</param>
/// <returns>a dictionary containing the evaluation results (default is false)</returns>
public static ItemRecommendationEvaluationResults Evaluate(
this IRecommender recommender,
IPosOnlyFeedback test,
IPosOnlyFeedback training,
IList<int> test_users = null,
IList<int> candidate_items = null,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
RepeatedEvents repeated_events = RepeatedEvents.No,
int n = -1)
{
if (test_users == null)
test_users = test.AllUsers;
candidate_items = Candidates(candidate_items, candidate_item_mode, test, training);
var result = new ItemRecommendationEvaluationResults();
// make sure that the user matrix is completely initialized before entering parallel code
var training_user_matrix = training.UserMatrix;
var test_user_matrix = test.UserMatrix;
int num_users = 0;
Parallel.ForEach(test_users, user_id => {
try
{
var correct_items = new HashSet<int>(test_user_matrix[user_id]);
correct_items.IntersectWith(candidate_items);
if (correct_items.Count == 0)
return;
var ignore_items_for_this_user = new HashSet<int>(
repeated_events == RepeatedEvents.Yes || training_user_matrix[user_id] == null ? new int[0] : training_user_matrix[user_id]
);
ignore_items_for_this_user.IntersectWith(candidate_items);
int num_candidates_for_this_user = candidate_items.Count - ignore_items_for_this_user.Count;
if (correct_items.Count == num_candidates_for_this_user)
return;
var prediction = recommender.Recommend(user_id, candidate_items:candidate_items, n:n, ignore_items:ignore_items_for_this_user);
var prediction_list = (from t in prediction select t.Item1).ToArray();
int num_dropped_items = num_candidates_for_this_user - prediction.Count;
double auc = AUC.Compute(prediction_list, correct_items, num_dropped_items);
double map = PrecisionAndRecall.AP(prediction_list, correct_items);
double ndcg = NDCG.Compute(prediction_list, correct_items);
double rr = ReciprocalRank.Compute(prediction_list, correct_items);
var positions = new int[] { 5, 10 };
var prec = PrecisionAndRecall.PrecisionAt(prediction_list, correct_items, positions);
var recall = PrecisionAndRecall.RecallAt(prediction_list, correct_items, positions);
// thread-safe incrementing
lock (result)
{
num_users++;
result["AUC"] += (float) auc;
result["MAP"] += (float) map;
result["NDCG"] += (float) ndcg;
result["MRR"] += (float) rr;
result["prec@5"] += (float) prec[5];
result["prec@10"] += (float) prec[10];
result["recall@5"] += (float) recall[5];
result["recall@10"] += (float) recall[10];
}
if (num_users % 1000 == 0)
Console.Error.Write(".");
if (num_users % 60000 == 0)
Console.Error.WriteLine();
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (string measure in Measures)
result[measure] /= num_users;
result["num_users"] = num_users;
result["num_lists"] = num_users;
result["num_items"] = candidate_items.Count;
return result;
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a dataset split</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public ItemRecommendationEvaluationResults DoCrossValidation(
this IRecommender recommender,
ISplit <IPosOnlyFeedback> split,
IList <int> test_users,
IList <int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var avg_results = new ItemRecommendationEvaluationResults();
if (!(recommender is ItemRecommender))
{
throw new ArgumentException("recommender must be of type ItemRecommender");
}
Parallel.For(0, (int)split.NumberOfFolds, fold =>
{
try
{
var split_recommender = (ItemRecommender)recommender.Clone(); // avoid changes in recommender
split_recommender.Feedback = split.Train[fold];
split_recommender.Train();
var fold_results = Items.Evaluate(split_recommender, split.Test[fold], split.Train[fold], test_users, candidate_items, candidate_item_mode);
if (compute_fit)
{
fold_results["fit"] = (float)split_recommender.ComputeFit();
}
// thread-safe stats
lock (avg_results)
foreach (var key in fold_results.Keys)
{
if (avg_results.ContainsKey(key))
{
avg_results[key] += fold_results[key];
}
else
{
avg_results[key] = fold_results[key];
}
}
if (show_results)
{
Console.Error.WriteLine("fold {0} {1}", fold, fold_results);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (var key in Items.Measures)
{
avg_results[key] /= split.NumberOfFolds;
}
avg_results["num_users"] /= split.NumberOfFolds;
avg_results["num_items"] /= split.NumberOfFolds;
return(avg_results);
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="num_folds">the number of folds</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
static public void DoRatingBasedRankingIterativeCrossValidation(
this RatingPredictor recommender,
uint num_folds,
IList<int> test_users,
IList<int> candidate_items,
CandidateItems candidate_item_mode,
RepeatedEvents repeated_events,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
var split = new RatingCrossValidationSplit(recommender.Ratings, num_folds);
recommender.DoRatingBasedRankingIterativeCrossValidation(split, test_users, candidate_items, candidate_item_mode, repeated_events, max_iter, find_iter);
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a positive-only feedback dataset split</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
public static void DoIterativeCrossValidation(
this IRecommender recommender,
ISplit<IPosOnlyFeedback> split,
IList<int> test_users,
IList<int> candidate_items,
CandidateItems candidate_item_mode,
RepeatedEvents repeated_events,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
if (!(recommender is IIterativeModel))
throw new ArgumentException("recommender must be of type IIterativeModel");
if (!(recommender is ItemRecommender))
throw new ArgumentException("recommender must be of type ItemRecommender");
var split_recommenders = new ItemRecommender[split.NumberOfFolds];
var iterative_recommenders = new IIterativeModel[split.NumberOfFolds];
var fold_results = new ItemRecommendationEvaluationResults[split.NumberOfFolds];
// initial training and evaluation
Parallel.For(0, (int) split.NumberOfFolds, i =>
{
try
{
split_recommenders[i] = (ItemRecommender) recommender.Clone(); // to avoid changes in recommender
split_recommenders[i].Feedback = split.Train[i];
split_recommenders[i].Train();
iterative_recommenders[i] = (IIterativeModel) split_recommenders[i];
fold_results[i] = Items.Evaluate(split_recommenders[i], split.Test[i], split.Train[i], test_users, candidate_items, candidate_item_mode, repeated_events);
if (show_fold_results)
Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, iterative_recommenders[i].NumIter);
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), iterative_recommenders[0].NumIter);
// iterative training and evaluation
for (int it = (int) iterative_recommenders[0].NumIter + 1; it <= max_iter; it++)
{
Parallel.For(0, (int) split.NumberOfFolds, i =>
{
try
{
iterative_recommenders[i].Iterate();
if (it % find_iter == 0)
{
fold_results[i] = Items.Evaluate(split_recommenders[i], split.Test[i], split.Train[i], test_users, candidate_items, candidate_item_mode, repeated_events);
if (show_fold_results)
Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, it);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), it);
}
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="num_folds">the number of folds</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
public static void DoIterativeCrossValidation(
this IRecommender recommender,
uint num_folds,
IList<int> test_users,
IList<int> candidate_items,
CandidateItems candidate_item_mode,
RepeatedEvents repeated_events,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
if (!(recommender is ItemRecommender))
throw new ArgumentException("recommender must be of type ItemRecommender");
var split = new PosOnlyFeedbackCrossValidationSplit<PosOnlyFeedback<SparseBooleanMatrix>>(((ItemRecommender) recommender).Feedback, num_folds);
recommender.DoIterativeCrossValidation(split, test_users, candidate_items, candidate_item_mode, repeated_events, max_iter, find_iter);
}
protected override void LoadData()
{
TimeSpan loading_time = Wrap.MeasureTime(delegate() {
base.LoadData();
// training data
training_data = double.IsNaN(rating_threshold)
? ItemData.Read(training_file, user_mapping, item_mapping, file_format == ItemDataFileFormat.IGNORE_FIRST_LINE)
: ItemDataRatingThreshold.Read(training_file, rating_threshold, user_mapping, item_mapping, file_format == ItemDataFileFormat.IGNORE_FIRST_LINE);
// test data
if (test_ratio == 0)
{
if (test_file != null)
{
test_data = double.IsNaN(rating_threshold)
? ItemData.Read(test_file, user_mapping, item_mapping, file_format == ItemDataFileFormat.IGNORE_FIRST_LINE)
: ItemDataRatingThreshold.Read(test_file, rating_threshold, user_mapping, item_mapping, file_format == ItemDataFileFormat.IGNORE_FIRST_LINE);
}
}
else
{
var split = new PosOnlyFeedbackSimpleSplit <PosOnlyFeedback <SparseBooleanMatrix> >(training_data, test_ratio);
training_data = split.Train[0];
test_data = split.Test[0];
}
if (user_prediction)
{
// swap file names for test users and candidate items
var ruf = test_users_file;
var rif = candidate_items_file;
test_users_file = rif;
candidate_items_file = ruf;
// swap user and item mappings
var um = user_mapping;
var im = item_mapping;
user_mapping = im;
item_mapping = um;
// transpose training and test data
training_data = training_data.Transpose();
// transpose test data
if (test_data != null)
{
test_data = test_data.Transpose();
}
}
for (int i = 0; i < recommenders.Count; i++)
{
if (recommenders[i] is MyMediaLite.ItemRecommendation.ItemRecommender)
{
((ItemRecommender)recommenders[i]).Feedback = training_data;
}
}
// test users
if (test_users_file != null)
{
test_users = user_mapping.ToInternalID(File.ReadLines(Path.Combine(data_dir, test_users_file)).ToArray());
}
else
{
test_users = test_data != null ? test_data.AllUsers : training_data.AllUsers;
}
// if necessary, perform user sampling
if (num_test_users > 0 && num_test_users < test_users.Count)
{
var old_test_users = new HashSet <int>(test_users);
var new_test_users = new int[num_test_users];
for (int i = 0; i < num_test_users; i++)
{
int random_index = MyMediaLite.Random.GetInstance().Next(old_test_users.Count - 1);
new_test_users[i] = old_test_users.ElementAt(random_index);
old_test_users.Remove(new_test_users[i]);
}
test_users = new_test_users;
}
// candidate items
if (candidate_items_file != null)
{
candidate_items = item_mapping.ToInternalID(File.ReadLines(Path.Combine(data_dir, candidate_items_file)).ToArray());
}
else if (all_items)
{
candidate_items = Enumerable.Range(0, item_mapping.InternalIDs.Max() + 1).ToArray();
}
if (candidate_items != null)
{
eval_item_mode = CandidateItems.EXPLICIT;
}
else if (in_training_items)
{
eval_item_mode = CandidateItems.TRAINING;
}
else if (in_test_items)
{
eval_item_mode = CandidateItems.TEST;
}
else if (overlap_items)
{
eval_item_mode = CandidateItems.OVERLAP;
}
else
{
eval_item_mode = CandidateItems.UNION;
}
});
//Salvar arquivos
List <string> linesToWrite = new List <string>();
for (int i = 0; i < training_data.UserMatrix.NumberOfRows; i++)
{
IList <int> columns = training_data.UserMatrix.GetEntriesByRow(i);
for (int j = 0; j < columns.Count; j++)
{
StringBuilder line = new StringBuilder();
line.Append(i.ToString() + " " + columns[j].ToString());
linesToWrite.Add(line.ToString());
}
}
System.IO.File.WriteAllLines("training.data", linesToWrite.ToArray());
linesToWrite = new List <string>();
for (int i = 0; i < test_data.UserMatrix.NumberOfRows; i++)
{
IList <int> columns = test_data.UserMatrix.GetEntriesByRow(i);
for (int j = 0; j < columns.Count; j++)
{
StringBuilder line = new StringBuilder();
line.Append(i.ToString() + " " + columns[j].ToString());
linesToWrite.Add(line.ToString());
}
}
System.IO.File.WriteAllLines("test.data", linesToWrite.ToArray());
/*
* List<string> linesToWrite = new List<string>();
* for (int rowIndex = 0; rowIndex < training_data.AllItems.Count; rowIndex++)
* {
*
* }*/
Console.Error.WriteLine(string.Format(CultureInfo.InvariantCulture, "loading_time {0,0:0.##}", loading_time.TotalSeconds));
Console.Error.WriteLine("memory {0}", Memory.Usage);
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a positive-only feedback dataset split</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
static public void DoRatingBasedRankingIterativeCrossValidation(
this RatingPredictor recommender,
ISplit <IRatings> split,
IList <int> test_users,
IList <int> candidate_items,
CandidateItems candidate_item_mode,
RepeatedEvents repeated_events,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
if (!(recommender is IIterativeModel))
{
throw new ArgumentException("recommender must be of type IIterativeModel");
}
var split_recommenders = new RatingPredictor[split.NumberOfFolds];
var iterative_recommenders = new IIterativeModel[split.NumberOfFolds];
var fold_results = new ItemRecommendationEvaluationResults[split.NumberOfFolds];
// initial training and evaluation
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
split_recommenders[i] = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender
split_recommenders[i].Ratings = split.Train[i];
split_recommenders[i].Train();
iterative_recommenders[i] = (IIterativeModel)split_recommenders[i];
var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]);
var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]);
fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events);
if (show_fold_results)
{
Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, iterative_recommenders[i].NumIter);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), iterative_recommenders[0].NumIter);
// iterative training and evaluation
for (int it = (int)iterative_recommenders[0].NumIter + 1; it <= max_iter; it++)
{
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
iterative_recommenders[i].Iterate();
if (it % find_iter == 0)
{
var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]);
var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]);
fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events);
if (show_fold_results)
{
Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, it);
}
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), it);
}
}
/// <summary>Evaluation for rankings of items</summary>
/// <remarks>
/// User-item combinations that appear in both sets are ignored for the test set, and thus in the evaluation,
/// except the boolean argument repeated_events is set.
///
/// The evaluation measures are listed in the Measures property.
/// Additionally, 'num_users' and 'num_items' report the number of users that were used to compute the results
/// and the number of items that were taken into account.
///
/// Literature:
/// <list type="bullet">
/// <item><description>
/// C. Manning, P. Raghavan, H. Schütze: Introduction to Information Retrieval, Cambridge University Press, 2008
/// </description></item>
/// </list>
///
/// On multi-core/multi-processor systems, the routine tries to use as many cores as possible,
/// which should to an almost linear speed-up.
/// </remarks>
/// <param name="recommender">item recommender</param>
/// <param name="test">test cases</param>
/// <param name="training">training data</param>
/// <param name="test_users">a list of integers with all test users; if null, use all users in the test cases</param>
/// <param name="candidate_items">a list of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <returns>a dictionary containing the evaluation results (default is false)</returns>
public static ItemRecommendationEvaluationResults Evaluate(
this IRecommender recommender,
IPosOnlyFeedback test,
IPosOnlyFeedback training,
IList<int> test_users = null,
IList<int> candidate_items = null,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool repeated_events = false)
{
switch (candidate_item_mode)
{
case CandidateItems.TRAINING: candidate_items = training.AllItems; break;
case CandidateItems.TEST: candidate_items = test.AllItems; break;
case CandidateItems.OVERLAP: candidate_items = new List<int>(test.AllItems.Intersect(training.AllItems)); break;
case CandidateItems.UNION: candidate_items = new List<int>(test.AllItems.Union(training.AllItems)); break;
}
if (candidate_items == null)
throw new ArgumentNullException("candidate_items");
if (test_users == null)
test_users = test.AllUsers;
int num_users = 0;
var result = new ItemRecommendationEvaluationResults();
// make sure that UserMatrix is completely initialized before entering parallel code
var training_user_matrix = training.UserMatrix;
var test_user_matrix = test.UserMatrix;
Parallel.ForEach(test_users, user_id =>
{
try
{
var correct_items = new HashSet<int>(test_user_matrix[user_id]);
correct_items.IntersectWith(candidate_items);
// the number of items that will be used for this user
var candidate_items_in_train = training_user_matrix[user_id] == null ? new HashSet<int>() : new HashSet<int>(training_user_matrix[user_id]);
candidate_items_in_train.IntersectWith(candidate_items);
int num_eval_items = candidate_items.Count - (repeated_events ? 0 : candidate_items_in_train.Count());
// skip all users that have 0 or #candidate_items test items
if (correct_items.Count == 0)
return;
if (num_eval_items == correct_items.Count)
return;
IList<int> prediction_list = recommender.PredictItems(user_id, candidate_items);
if (prediction_list.Count != candidate_items.Count)
throw new Exception("Not all items have been ranked.");
ICollection<int> ignore_items = (repeated_events || training_user_matrix[user_id] == null) ? new int[0] : training_user_matrix[user_id];
double auc = AUC.Compute(prediction_list, correct_items, ignore_items);
double map = PrecisionAndRecall.AP(prediction_list, correct_items, ignore_items);
double ndcg = NDCG.Compute(prediction_list, correct_items, ignore_items);
double rr = ReciprocalRank.Compute(prediction_list, correct_items, ignore_items);
var positions = new int[] { 3, 5, 10 }; // DH: added for p@3 & r@3
var prec = PrecisionAndRecall.PrecisionAt(prediction_list, correct_items, ignore_items, positions);
var recall = PrecisionAndRecall.RecallAt(prediction_list, correct_items, ignore_items, positions);
// thread-safe incrementing
lock (result)
{
num_users++;
result["AUC"] += (float)auc;
result["MAP"] += (float)map;
result["NDCG"] += (float)ndcg;
result["MRR"] += (float)rr;
result["prec@3"] += (float)prec[3];
result["prec@5"] += (float)prec[5];
result["prec@10"] += (float)prec[10];
result["recall@3"] += (float)recall[3];
result["recall@5"] += (float)recall[5];
result["recall@10"] += (float)recall[10];
}
if (num_users % 1000 == 0)
Console.Error.Write(".");
if (num_users % 60000 == 0)
Console.Error.WriteLine();
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
Console.Error.WriteLine("===> ERROR: user_id=" + user_id);
Console.Error.WriteLine("===> ERROR: training_user_matrix[user_id]=" + training_user_matrix[user_id]);
throw e;
}
});
foreach (string measure in Measures)
result[measure] /= num_users;
result["num_users"] = num_users;
result["num_lists"] = num_users;
result["num_items"] = candidate_items.Count;
return result;
}
// TODO consider micro- (by item) and macro-averaging (by user, the current thing); repeated events
/// <summary>Online evaluation for rankings of items</summary>
/// <remarks>
/// </remarks>
/// <param name="recommender">the item recommender to be evaluated</param>
/// <param name="test">test cases</param>
/// <param name="training">training data (must be connected to the recommender's training data)</param>
/// <param name="test_users">a list of all test user IDs</param>
/// <param name="candidate_items">a list of all candidate item IDs</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <returns>a dictionary containing the evaluation results (averaged by user)</returns>
public static ItemRecommendationEvaluationResults EvaluateOnline(
this IRecommender recommender,
IPosOnlyFeedback test, IPosOnlyFeedback training,
IList<int> test_users, IList<int> candidate_items,
CandidateItems candidate_item_mode)
{
var incremental_recommender = recommender as IIncrementalItemRecommender;
if (incremental_recommender == null)
throw new ArgumentException("recommender must be of type IIncrementalItemRecommender");
// prepare candidate items once to avoid recreating them
switch (candidate_item_mode)
{
case CandidateItems.TRAINING: candidate_items = training.AllItems; break;
case CandidateItems.TEST: candidate_items = test.AllItems; break;
case CandidateItems.OVERLAP: candidate_items = new List<int>(test.AllItems.Intersect(training.AllItems)); break;
case CandidateItems.UNION: candidate_items = new List<int>(test.AllItems.Union(training.AllItems)); break;
}
candidate_item_mode = CandidateItems.EXPLICIT;
// for better handling, move test data points into arrays
var users = new int[test.Count];
var items = new int[test.Count];
int pos = 0;
foreach (int user_id in test.UserMatrix.NonEmptyRowIDs)
foreach (int item_id in test.UserMatrix[user_id])
{
users[pos] = user_id;
items[pos] = item_id;
pos++;
}
// random order of the test data points // TODO chronological order
var random_index = new int[test.Count];
for (int index = 0; index < random_index.Length; index++)
random_index[index] = index;
random_index.Shuffle();
var results_by_user = new Dictionary<int, ItemRecommendationEvaluationResults>();
int num_lists = 0;
foreach (int index in random_index)
{
if (test_users.Contains(users[index]) && candidate_items.Contains(items[index]))
{
// evaluate user
var current_test = new PosOnlyFeedback<SparseBooleanMatrix>();
current_test.Add(users[index], items[index]);
var current_result = Items.Evaluate(recommender, current_test, training, current_test.AllUsers, candidate_items, candidate_item_mode);
if (current_result["num_users"] == 1)
if (results_by_user.ContainsKey(users[index]))
{
foreach (string measure in Items.Measures)
results_by_user[users[index]][measure] += current_result[measure];
results_by_user[users[index]]["num_items"]++;
num_lists++;
}
else
{
results_by_user[users[index]] = current_result;
results_by_user[users[index]]["num_items"] = 1;
results_by_user[users[index]].Remove("num_users");
}
}
// update recommender
incremental_recommender.AddFeedback(users[index], items[index]);
}
var results = new ItemRecommendationEvaluationResults();
foreach (int u in results_by_user.Keys)
foreach (string measure in Items.Measures)
results[measure] += results_by_user[u][measure] / results_by_user[u]["num_items"];
foreach (string measure in Items.Measures)
results[measure] /= results_by_user.Count;
results["num_users"] = results_by_user.Count;
results["num_items"] = candidate_items.Count;
results["num_lists"] = num_lists;
return results;
}
static void LoadData()
{
TimeSpan loading_time = Wrap.MeasureTime(delegate() {
// training data
training_file = Path.Combine(data_dir, training_file);
training_data = double.IsNaN(rating_threshold)
? ItemData.Read(training_file, user_mapping, item_mapping, file_format == ItemDataFileFormat.IGNORE_FIRST_LINE)
: ItemDataRatingThreshold.Read(training_file, rating_threshold, user_mapping, item_mapping, file_format == ItemDataFileFormat.IGNORE_FIRST_LINE);
// user attributes
if (user_attributes_file != null)
user_attributes = AttributeData.Read(Path.Combine(data_dir, user_attributes_file), user_mapping);
if (recommender is IUserAttributeAwareRecommender)
((IUserAttributeAwareRecommender)recommender).UserAttributes = user_attributes;
// item attributes
if (item_attributes_file != null)
item_attributes = AttributeData.Read(Path.Combine(data_dir, item_attributes_file), item_mapping);
if (recommender is IItemAttributeAwareRecommender)
((IItemAttributeAwareRecommender)recommender).ItemAttributes = item_attributes;
// user relation
if (recommender is IUserRelationAwareRecommender)
{
((IUserRelationAwareRecommender)recommender).UserRelation = RelationData.Read(Path.Combine(data_dir, user_relations_file), user_mapping);
Console.WriteLine("relation over {0} users", ((IUserRelationAwareRecommender)recommender).NumUsers);
}
// item relation
if (recommender is IItemRelationAwareRecommender)
{
((IItemRelationAwareRecommender)recommender).ItemRelation = RelationData.Read(Path.Combine(data_dir, item_relations_file), item_mapping);
Console.WriteLine("relation over {0} items", ((IItemRelationAwareRecommender)recommender).NumItems);
}
// user groups
if (user_groups_file != null)
{
group_to_user = RelationData.Read(Path.Combine(data_dir, user_groups_file), user_mapping); // assumption: user and user group IDs are disjoint
user_groups = group_to_user.NonEmptyRowIDs;
Console.WriteLine("{0} user groups", user_groups.Count);
}
// test data
if (test_ratio == 0)
{
if (test_file != null)
{
test_file = Path.Combine(data_dir, test_file);
test_data = double.IsNaN(rating_threshold)
? ItemData.Read(test_file, user_mapping, item_mapping, file_format == ItemDataFileFormat.IGNORE_FIRST_LINE)
: ItemDataRatingThreshold.Read(test_file, rating_threshold, user_mapping, item_mapping, file_format == ItemDataFileFormat.IGNORE_FIRST_LINE);
}
}
else
{
var split = new PosOnlyFeedbackSimpleSplit<PosOnlyFeedback<SparseBooleanMatrix>>(training_data, test_ratio);
training_data = split.Train[0];
test_data = split.Test[0];
}
if (group_method == "GroupsAsUsers")
{
Console.WriteLine("group recommendation strategy: {0}", group_method);
// TODO verify what is going on here
//var training_data_group = new PosOnlyFeedback<SparseBooleanMatrix>();
// transform groups to users
foreach (int group_id in group_to_user.NonEmptyRowIDs)
foreach (int user_id in group_to_user[group_id])
foreach (int item_id in training_data.UserMatrix.GetEntriesByRow(user_id))
training_data.Add(group_id, item_id);
// add the users that do not belong to groups
//training_data = training_data_group;
// transform groups to users
var test_data_group = new PosOnlyFeedback<SparseBooleanMatrix>();
foreach (int group_id in group_to_user.NonEmptyRowIDs)
foreach (int user_id in group_to_user[group_id])
foreach (int item_id in test_data.UserMatrix.GetEntriesByRow(user_id))
test_data_group.Add(group_id, item_id);
test_data = test_data_group;
group_method = null; // deactivate s.t. the normal eval routines are used
}
if (user_prediction)
{
// swap file names for test users and candidate items
var ruf = test_users_file;
var rif = candidate_items_file;
test_users_file = rif;
candidate_items_file = ruf;
// swap user and item mappings
var um = user_mapping;
var im = item_mapping;
user_mapping = im;
item_mapping = um;
// transpose training and test data
training_data = training_data.Transpose();
// transpose test data
if (test_data != null)
test_data = test_data.Transpose();
}
if (recommender is MyMediaLite.ItemRecommendation.ItemRecommender)
((ItemRecommender)recommender).Feedback = training_data;
// test users
if (test_users_file != null)
test_users = user_mapping.ToInternalID( File.ReadLines(Path.Combine(data_dir, test_users_file)).ToArray() );
else
test_users = test_data != null ? test_data.AllUsers : training_data.AllUsers;
// if necessary, perform user sampling
if (num_test_users > 0 && num_test_users < test_users.Count)
{
var old_test_users = new HashSet<int>(test_users);
var new_test_users = new int[num_test_users];
for (int i = 0; i < num_test_users; i++)
{
int random_index = MyMediaLite.Util.Random.GetInstance().Next(old_test_users.Count - 1);
new_test_users[i] = old_test_users.ElementAt(random_index);
old_test_users.Remove(new_test_users[i]);
}
test_users = new_test_users;
}
// candidate items
if (candidate_items_file != null)
candidate_items = item_mapping.ToInternalID( File.ReadLines(Path.Combine(data_dir, candidate_items_file)).ToArray() );
else if (all_items)
candidate_items = Enumerable.Range(0, item_mapping.InternalIDs.Max() + 1).ToArray();
if (candidate_items != null)
eval_item_mode = CandidateItems.EXPLICIT;
else if (in_training_items)
eval_item_mode = CandidateItems.TRAINING;
else if (in_test_items)
eval_item_mode = CandidateItems.TEST;
else if (overlap_items)
eval_item_mode = CandidateItems.OVERLAP;
else
eval_item_mode = CandidateItems.UNION;
});
Console.Error.WriteLine(string.Format(CultureInfo.InvariantCulture, "loading_time {0,0:0.##}", loading_time.TotalSeconds));
Console.Error.WriteLine("memory {0}", Memory.Usage);
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a dataset split</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public EvaluationResults DoRatingBasedRankingCrossValidation(
this RatingPredictor recommender,
ISplit <IRatings> split,
IList <int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var avg_results = new ItemRecommendationEvaluationResults();
Parallel.For(0, (int)split.NumberOfFolds, fold =>
{
try
{
var split_recommender = (RatingPredictor)recommender.Clone(); // avoid changes in recommender
split_recommender.Ratings = split.Train[fold];
split_recommender.Train();
var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[fold]);
var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[fold]);
IList <int> test_users = test_data_posonly.AllUsers;
var fold_results = Items.Evaluate(split_recommender, test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode);
if (compute_fit)
{
fold_results["fit"] = (float)split_recommender.ComputeFit();
}
// thread-safe stats
lock (avg_results)
foreach (var key in fold_results.Keys)
{
if (avg_results.ContainsKey(key))
{
avg_results[key] += fold_results[key];
}
else
{
avg_results[key] = fold_results[key];
}
}
if (show_results)
{
Console.Error.WriteLine("fold {0} {1}", fold, fold_results);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (var key in Items.Measures)
{
avg_results[key] /= split.NumberOfFolds;
}
avg_results["num_users"] /= split.NumberOfFolds;
avg_results["num_items"] /= split.NumberOfFolds;
return(avg_results);
}
