/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public Dictionary <string, double> EvaluateOnSplit(RatingPredictor recommender, ISplit <IRatings> split, bool show_results)
{
var avg_results = new Dictionary <string, double>();
foreach (var key in Measures)
{
avg_results[key] = 0;
}
for (int i = 0; i < split.NumberOfFolds; i++)
{
var split_recommender = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender
split_recommender.Ratings = split.Train[i];
split_recommender.Train();
var fold_results = Evaluate(split_recommender, split.Test[i]);
foreach (var key in fold_results.Keys)
{
avg_results[key] += fold_results[key];
}
if (show_results)
{
Console.Error.WriteLine("fold {0}, RMSE {1,0:0.#####}, MAE {2,0:0.#####}", i, fold_results["RMSE"].ToString(CultureInfo.InvariantCulture), fold_results["MAE"].ToString(CultureInfo.InvariantCulture));
}
}
foreach (var key in avg_results.Keys.ToList())
{
avg_results[key] /= split.NumberOfFolds;
}
return(avg_results);
}
/// <summary>Performs user-wise fold-in evaluation, but instead of folding in perform a complete re-training with the new data</summary>
/// <remarks>
/// This method can be quite slow.
/// </remarks>
/// <returns>the evaluation results</returns>
/// <param name='recommender'>a rating predictor capable of performing a user fold-in</param>
/// <param name='update_data'>the rating data used to represent the users</param>
/// <param name='eval_data'>the evaluation data</param>
static public RatingPredictionEvaluationResults EvaluateFoldInCompleteRetraining(this RatingPredictor recommender, IRatings update_data, IRatings eval_data)
{
double rmse = 0;
double mae = 0;
double cbd = 0;
int rating_count = 0;
foreach (int user_id in update_data.AllUsers)
{
if (eval_data.AllUsers.Contains(user_id))
{
var local_recommender = (RatingPredictor)recommender.Clone();
var known_ratings = new RatingsProxy(update_data, update_data.ByUser[user_id]);
local_recommender.Ratings = new CombinedRatings(recommender.Ratings, known_ratings);
local_recommender.Train();
var items_to_rate = (from index in eval_data.ByUser[user_id] select eval_data.Items[index]).ToArray();
var predicted_ratings = recommender.Recommend(user_id, candidate_items: items_to_rate);
foreach (var pred in predicted_ratings)
{
float prediction = pred.Item2;
float actual_rating = eval_data.Get(user_id, pred.Item1, eval_data.ByUser[user_id]);
float error = prediction - actual_rating;
rmse += error * error;
mae += Math.Abs(error);
cbd += Eval.Ratings.ComputeCBD(actual_rating, prediction, recommender.MinRating, recommender.MaxRating);
rating_count++;
}
Console.Error.Write(".");
}
}
mae = mae / rating_count;
rmse = Math.Sqrt(rmse / rating_count);
cbd = cbd / rating_count;
var result = new RatingPredictionEvaluationResults();
result["RMSE"] = (float)rmse;
result["MAE"] = (float)mae;
result["NMAE"] = (float)mae / (recommender.MaxRating - recommender.MinRating);
result["CBD"] = (float)cbd;
return(result);
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_fold_results">set to true to print per-fold results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public RatingPredictionEvaluationResults DoCrossValidation(
this RatingPredictor recommender,
ISplit <IRatings> split,
bool compute_fit = false,
bool show_fold_results = false)
{
var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds];
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
var split_recommender = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender
split_recommender.Ratings = split.Train[i];
if (recommender is ITransductiveRatingPredictor)
{
((ITransductiveRatingPredictor)split_recommender).AdditionalFeedback = split.Test[i];
}
split_recommender.Train();
fold_results[i] = Ratings.Evaluate(split_recommender, split.Test[i]);
if (compute_fit)
{
fold_results[i]["fit"] = (float)split_recommender.ComputeFit();
}
if (show_fold_results)
{
Console.Error.WriteLine("fold {0} {1}", i, fold_results[i]);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
return(new RatingPredictionEvaluationResults(fold_results));
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">a rating predictor</param>
/// <param name="split">a rating dataset split</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
static public void DoIterativeCrossValidation(
this RatingPredictor recommender,
ISplit <IRatings> split,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
if (!(recommender is IIterativeModel))
{
throw new ArgumentException("recommender must be of type IIterativeModel");
}
var split_recommenders = new RatingPredictor[split.NumberOfFolds];
var iterative_recommenders = new IIterativeModel[split.NumberOfFolds];
var fold_results = new RatingPredictionEvaluationResults[split.NumberOfFolds];
// initial training and evaluation
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
split_recommenders[i] = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender
split_recommenders[i].Ratings = split.Train[i];
if (recommender is ITransductiveRatingPredictor)
{
((ITransductiveRatingPredictor)split_recommenders[i]).AdditionalFeedback = split.Test[i];
}
split_recommenders[i].Train();
iterative_recommenders[i] = (IIterativeModel)split_recommenders[i];
fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]);
if (show_fold_results)
{
Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], iterative_recommenders[i].NumIter);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), iterative_recommenders[0].NumIter);
// iterative training and evaluation
for (int it = (int)iterative_recommenders[0].NumIter + 1; it <= max_iter; it++)
{
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
iterative_recommenders[i].Iterate();
if (it % find_iter == 0)
{
fold_results[i] = Ratings.Evaluate(split_recommenders[i], split.Test[i]);
if (show_fold_results)
{
Console.Error.WriteLine("fold {0} {1} iteration {2}", i, fold_results[i], it);
}
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new RatingPredictionEvaluationResults(fold_results), it);
}
}
/// <summary>Evaluate on the folds of a dataset split</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a dataset split</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="compute_fit">if set to true measure fit on the training data as well</param>
/// <param name="show_results">set to true to print results to STDERR</param>
/// <returns>a dictionary containing the average results over the different folds of the split</returns>
static public EvaluationResults DoRatingBasedRankingCrossValidation(
this RatingPredictor recommender,
ISplit <IRatings> split,
IList <int> candidate_items,
CandidateItems candidate_item_mode = CandidateItems.OVERLAP,
bool compute_fit = false,
bool show_results = false)
{
var avg_results = new ItemRecommendationEvaluationResults();
Parallel.For(0, (int)split.NumberOfFolds, fold =>
{
try
{
var split_recommender = (RatingPredictor)recommender.Clone(); // avoid changes in recommender
split_recommender.Ratings = split.Train[fold];
split_recommender.Train();
var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[fold]);
var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[fold]);
IList <int> test_users = test_data_posonly.AllUsers;
var fold_results = Items.Evaluate(split_recommender, test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode);
if (compute_fit)
{
fold_results["fit"] = (float)split_recommender.ComputeFit();
}
// thread-safe stats
lock (avg_results)
foreach (var key in fold_results.Keys)
{
if (avg_results.ContainsKey(key))
{
avg_results[key] += fold_results[key];
}
else
{
avg_results[key] = fold_results[key];
}
}
if (show_results)
{
Console.Error.WriteLine("fold {0} {1}", fold, fold_results);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
foreach (var key in Items.Measures)
{
avg_results[key] /= split.NumberOfFolds;
}
avg_results["num_users"] /= split.NumberOfFolds;
avg_results["num_items"] /= split.NumberOfFolds;
return(avg_results);
}
/// <summary>Evaluate an iterative recommender on the folds of a dataset split, display results on STDOUT</summary>
/// <param name="recommender">an item recommender</param>
/// <param name="split">a positive-only feedback dataset split</param>
/// <param name="test_users">a collection of integers with all test users</param>
/// <param name="candidate_items">a collection of integers with all candidate items</param>
/// <param name="candidate_item_mode">the mode used to determine the candidate items</param>
/// <param name="repeated_events">allow repeated events in the evaluation (i.e. items accessed by a user before may be in the recommended list)</param>
/// <param name="max_iter">the maximum number of iterations</param>
/// <param name="find_iter">the report interval</param>
/// <param name="show_fold_results">if set to true to print per-fold results to STDERR</param>
static public void DoRatingBasedRankingIterativeCrossValidation(
this RatingPredictor recommender,
ISplit <IRatings> split,
IList <int> test_users,
IList <int> candidate_items,
CandidateItems candidate_item_mode,
RepeatedEvents repeated_events,
uint max_iter,
uint find_iter = 1,
bool show_fold_results = false)
{
if (!(recommender is IIterativeModel))
{
throw new ArgumentException("recommender must be of type IIterativeModel");
}
var split_recommenders = new RatingPredictor[split.NumberOfFolds];
var iterative_recommenders = new IIterativeModel[split.NumberOfFolds];
var fold_results = new ItemRecommendationEvaluationResults[split.NumberOfFolds];
// initial training and evaluation
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
split_recommenders[i] = (RatingPredictor)recommender.Clone(); // to avoid changes in recommender
split_recommenders[i].Ratings = split.Train[i];
split_recommenders[i].Train();
iterative_recommenders[i] = (IIterativeModel)split_recommenders[i];
var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]);
var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]);
fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events);
if (show_fold_results)
{
Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, iterative_recommenders[i].NumIter);
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), iterative_recommenders[0].NumIter);
// iterative training and evaluation
for (int it = (int)iterative_recommenders[0].NumIter + 1; it <= max_iter; it++)
{
Parallel.For(0, (int)split.NumberOfFolds, i =>
{
try
{
iterative_recommenders[i].Iterate();
if (it % find_iter == 0)
{
var test_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Test[i]);
var training_data_posonly = new PosOnlyFeedback <SparseBooleanMatrix>(split.Train[i]);
fold_results[i] = Items.Evaluate(split_recommenders[i], test_data_posonly, training_data_posonly, test_users, candidate_items, candidate_item_mode, repeated_events);
if (show_fold_results)
{
Console.WriteLine("fold {0} {1} iteration {2}", i, fold_results, it);
}
}
}
catch (Exception e)
{
Console.Error.WriteLine("===> ERROR: " + e.Message + e.StackTrace);
throw;
}
});
Console.WriteLine("{0} iteration {1}", new ItemRecommendationEvaluationResults(fold_results), it);
}
}
