/// <summary>Predict ratings (double precision)</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="ratings">the ratings to predict</param>
/// <param name="writer">the writer object to write the predictions to</param>
public static void PredictRatingsDouble(IRecommender recommender, IRatings ratings, BinaryWriter writer)
{
for (int i = 0; i < ratings.Count; i++)
{
writer.Write(recommender.Predict(ratings.Users[i], ratings.Items[i]).ToString(CultureInfo.InvariantCulture));
}
}
public static void Evaluate(IRecommender recommender,
IDataModel model,
int samples,
IRunningAverage tracker,
String tag)
{
printHeader();
var users = recommender.GetDataModel().GetUserIDs();
while (users.MoveNext())
{
long userID = users.Current;
var recs1 = recommender.Recommend(userID, model.GetNumItems());
IPreferenceArray prefs2 = model.GetPreferencesFromUser(userID);
prefs2.SortByValueReversed();
FastIDSet commonSet = new FastIDSet();
long maxItemID = setBits(commonSet, recs1, samples);
FastIDSet otherSet = new FastIDSet();
maxItemID = Math.Max(maxItemID, setBits(otherSet, prefs2, samples));
int max = mask(commonSet, otherSet, maxItemID);
max = Math.Min(max, samples);
if (max < 2)
{
continue;
}
long[] items1 = getCommonItems(commonSet, recs1, max);
long[] items2 = getCommonItems(commonSet, prefs2, max);
double variance = scoreCommonSubset(tag, userID, samples, max, items1, items2);
tracker.AddDatum(variance);
}
}
/// <summary>List all recommenders in a given namespace</summary>
/// <param name="prefix">a string representing the namespace</param>
/// <returns>an array of strings containing the recommender descriptions</returns>
public static IList <string> ListRecommenders(this string prefix)
{
var result = new List <string>();
foreach (Type type in Utils.GetTypes(prefix))
{
if (!type.IsAbstract && !type.IsInterface && !type.IsEnum && !type.IsGenericType && type.GetInterface("IRecommender") != null)
{
IRecommender recommender = prefix.Equals("MyMediaLite.RatingPrediction") ? (IRecommender)type.CreateRatingPredictor() : (IRecommender)type.CreateItemRecommender();
string description = recommender.ToString();
string needs = recommender.Needs();
if (needs.Length > 0)
{
description += "\n needs " + needs;
}
string supports = recommender.Supports();
if (supports.Length > 0)
{
description += "\n supports " + supports;
}
result.Add(description);
}
}
return(result);
}
/// <summary>Predict items for Track 2</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="candidates">a mapping from user IDs to the candidate items</param>
/// <param name="writer">the writer object to write the predictions to</param>
public static void PredictTrack2(IRecommender recommender, Dictionary <int, IList <int> > candidates, TextWriter writer)
{
foreach (int user_id in candidates.Keys) // this is ordered, but is it guaranteed?
{
IList <int> user_candidates = candidates[user_id];
var predictions = new double[user_candidates.Count];
for (int i = 0; i < user_candidates.Count; i++)
{
predictions[i] = recommender.Predict(user_id, user_candidates[i]);
}
var positions = new List <int>(new int[] { 0, 1, 2, 3, 4, 5 });
positions.Sort(delegate(int pos1, int pos2) { return(predictions[pos2].CompareTo(predictions[pos1])); });
for (int i = 0; i < user_candidates.Count; i++)
{
if (positions.IndexOf(i) < 3)
{
writer.Write("1");
}
else
{
writer.Write("0");
}
}
}
}
/// <summary>Describes the kind of arguments supported by this recommender</summary>
/// <param name="recommender">a recommender</param>
/// <returns>a string containing the additional arguments supported by this recommender</returns>
public static string Supports(this IRecommender recommender)
{
// determine necessary data
var supports = new List <string>();
/*
* if (recommender is IUserSimilarityProvider)
* needs.Add("");
* if (recommender is IItemSimilarityProvider)
* needs.Add("");
*/
if (recommender is IIterativeModel)
{
supports.Add("--find-iter=N");
}
if (recommender is IIncrementalItemRecommender)
{
supports.Add("--online-evaluation");
}
if (recommender is IIncrementalRatingPredictor)
{
supports.Add("--online-evaluation");
}
return(string.Join(", ", supports.ToArray()));
}
/// <summary>Save the model parameters of a recommender (in a given iteration of the training) to a file</summary>
/// <remarks>
/// Does not save if filename is an empty string.
/// </remarks>
/// <param name="recommender">the <see cref="IRecommender"/> to save</param>
/// <param name="filename">the filename template</param>
/// <param name="iteration">the iteration (will be appended to the filename)</param>
public static void Save(IRecommender recommender, string filename, int iteration)
{
if (filename == null)
return;
Save(recommender, filename + "-it-" + iteration);
}
/// <summary>Evaluate Track 2 on a validation set</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="candidates">the candidate items (per user)</param>
/// <param name="hits">the real items (per user)</param>
/// <returns>the error rate on this validation split</returns>
public static double EvaluateTrack2(IRecommender recommender, Dictionary <int, IList <int> > candidates, Dictionary <int, IList <int> > hits)
{
int hit_count = 0;
foreach (int user_id in candidates.Keys)
{
IList <int> user_candidates = candidates[user_id];
var predictions = new double[user_candidates.Count];
for (int i = 0; i < user_candidates.Count; i++)
{
predictions[i] = recommender.Predict(user_id, user_candidates[i]);
}
var positions = new List <int>(new int[] { 0, 1, 2, 3, 4, 5 });
positions.Sort(delegate(int pos1, int pos2) { return(predictions[pos2].CompareTo(predictions[pos1])); });
var user_true_items = new HashSet <int>(hits[user_id]);
for (int i = 0; i < user_true_items.Count; i++)
{
if (user_true_items.Contains(user_candidates[positions[i]]))
{
hit_count++;
}
}
}
int num_pos = hits.Keys.Sum(u => hits[u].Count);
return(1 - (double)hit_count / num_pos);
}
public static void Evaluate(IRecommender recommender,
IDataModel model,
int samples,
IRunningAverage tracker,
String tag) {
printHeader();
var users = recommender.GetDataModel().GetUserIDs();
while (users.MoveNext()) {
long userID = users.Current;
var recs1 = recommender.Recommend(userID, model.GetNumItems());
IPreferenceArray prefs2 = model.GetPreferencesFromUser(userID);
prefs2.SortByValueReversed();
FastIDSet commonSet = new FastIDSet();
long maxItemID = setBits(commonSet, recs1, samples);
FastIDSet otherSet = new FastIDSet();
maxItemID = Math.Max(maxItemID, setBits(otherSet, prefs2, samples));
int max = mask(commonSet, otherSet, maxItemID);
max = Math.Min(max, samples);
if (max < 2) {
continue;
}
long[] items1 = getCommonItems(commonSet, recs1, max);
long[] items2 = getCommonItems(commonSet, prefs2, max);
double variance = scoreCommonSubset(tag, userID, samples, max, items1, items2);
tracker.AddDatum(variance);
}
}
public MainForm()
{
InitializeComponent();
IRater rate = new LinearRater(-4, 2, 3, 1);
IComparer compare = new CorrelationUserComparer();
recommender = new UserCollaborativeFilterRecommender(compare, rate, 50);
if (File.Exists(savedModel))
{
try
{
recommender.Load(savedModel);
rtbOutput.Text = "Loaded model from file";
EnableForm(true);
}
catch
{
rtbOutput.Text = "Saved model is corrupt";
}
}
//RecommenderTests.TestAllRecommenders();
//RecommenderTests.FindBestRaterWeights();
}
public static ScoreResults Score(this IRecommender classifier, UserBehaviorDatabase db, IRater rater)
{
UserBehaviorTransformer ubt = new UserBehaviorTransformer(db);
UserArticleRatingsTable actualRatings = ubt.GetUserArticleRatingsTable(rater);
var distinctUserArticlePairs = db.UserActions.GroupBy(x => new { x.UserID, x.ArticleID }).ToList();
double score = 0.0;
int count = 0;
foreach (var userArticle in distinctUserArticlePairs)
{
int userIndex = actualRatings.UserIndexToID.IndexOf(userArticle.Key.UserID);
int articleIndex = actualRatings.ArticleIndexToID.IndexOf(userArticle.Key.ArticleID);
double actualRating = actualRatings.Users[userIndex].ArticleRatings[articleIndex];
if (actualRating != 0)
{
double predictedRating = classifier.GetRating(userArticle.Key.UserID, userArticle.Key.ArticleID);
score += Math.Pow(predictedRating - actualRating, 2);
count++;
}
}
if (count > 0)
{
score = Math.Sqrt(score / count);
}
return(new ScoreResults(score));
}
/// <summary>Write item predictions (scores) to a TextWriter object</summary>
/// <param name="recommender">the <see cref="IRecommender"/> to use for making the predictions</param>
/// <param name="train">a user-wise <see cref="IPosOnlyFeedback"/> containing the items already observed</param>
/// <param name="candidate_items">list of candidate items</param>
/// <param name="num_predictions">number of items to return per user, -1 if there should be no limit</param>
/// <param name="writer">the <see cref="TextWriter"/> to write to</param>
/// <param name="users">a list of users to make recommendations for; if null, all users in train will be provided with recommendations</param>
/// <param name="user_mapping">an <see cref="IMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="IMapping"/> object for the item IDs</param>
/// <param name="repeated_items">true if items that a user has already accessed shall also be predicted</param>
static public void WritePredictions(
this IRecommender recommender,
IPosOnlyFeedback train,
ICollection <int> candidate_items,
int num_predictions,
TextWriter writer,
IList <int> users = null,
IMapping user_mapping = null, IMapping item_mapping = null,
bool repeated_items = false)
{
if (users == null)
{
users = new List <int>(train.AllUsers);
}
ICollection <int> ignore_items = new int[0];
foreach (int user_id in users)
{
if (!repeated_items)
{
ignore_items = train.UserMatrix[user_id];
}
WritePredictions(recommender, user_id, candidate_items, ignore_items, num_predictions, writer, user_mapping, item_mapping);
}
}
/// <summary>Evaluate Track 2 on a validation set</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="candidates">the candidate items (per user)</param>
/// <param name="hits">the real items (per user)</param>
/// <returns>the error rate on this validation split</returns>
public static double EvaluateTrack2(IRecommender recommender, Dictionary<int, IList<int>> candidates, Dictionary<int, IList<int>> hits)
{
int hit_count = 0;
foreach (int user_id in candidates.Keys)
{
IList<int> user_candidates = candidates[user_id];
var predictions = new double[user_candidates.Count];
for (int i = 0; i < user_candidates.Count; i++)
predictions[i] = recommender.Predict(user_id, user_candidates[i]);
var positions = new List<int>(new int[] { 0, 1, 2, 3, 4, 5 });
positions.Sort(delegate(int pos1, int pos2) { return predictions[pos2].CompareTo(predictions[pos1]); } );
var user_true_items = new HashSet<int>(hits[user_id]);
for (int i = 0; i < user_true_items.Count; i++)
if (user_true_items.Contains(user_candidates[positions[i]]))
hit_count++;
}
int num_pos = hits.Keys.Sum(u => hits[u].Count);
return 1 - (double) hit_count / num_pos;
}
/// <summary>Save the model parameters of a recommender to a file</summary>
/// <remarks>
/// Does not save if filename is an empty string.
/// </remarks>
/// <param name="recommender">the recommender to store</param>
/// <param name="filename">the filename (may include relative paths)</param>
public static void Save(IRecommender recommender, string filename)
{
if (filename == null)
return;
Console.Error.WriteLine("Save model to {0}", filename);
recommender.SaveModel(filename);
}
/// <summary>Save the model parameters of a recommender (in a given iteration of the training) to a file</summary>
/// <remarks>
/// Does not save if filename is an empty string.
/// </remarks>
/// <param name="recommender">the <see cref="IRecommender"/> to save</param>
/// <param name="filename">the filename template</param>
/// <param name="iteration">the iteration (will be appended to the filename)</param>
public static void SaveModel(IRecommender recommender, string filename, int iteration)
{
if (filename == string.Empty)
{
return;
}
SaveModel(recommender, filename + "-it-" + iteration);
}
/// <summary>Save the model parameters of a recommender (in a given iteration of the training) to a file</summary>
/// <remarks>
/// Does not save if filename is an empty string.
/// </remarks>
/// <param name="recommender">the <see cref="IRecommender"/> to save</param>
/// <param name="filename">the filename template</param>
/// <param name="iteration">the iteration (will be appended to the filename)</param>
public static void Save(IRecommender recommender, string filename, int iteration)
{
if (filename == null)
{
return;
}
Save(recommender, filename + "-it-" + iteration);
}
/// <summary>Predict ratings for Track 1</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="ratings">the ratings to predict</param>
/// <param name="writer">the writer object to write the predictions to</param>
public static void PredictRatings(IRecommender recommender, IRatings ratings, BinaryWriter writer)
{
for (int i = 0; i < ratings.Count; i++)
{
double prediction = recommender.Predict(ratings.Users[i], ratings.Items[i]);
byte encoded_prediction = (byte)(2.55 * prediction + 0.5);
writer.Write(encoded_prediction);
}
}
public ChatController(IChatService chatService, UserManager <User> userManager, IMapper mapper, IEmotionGetter emotionGetter, IMoodService moodService, IRecommender recommender, IRecommendationService recommendationService)
{
this.chatService = chatService;
this.userManager = userManager;
this.mapper = mapper;
this.emotionGetter = emotionGetter;
this.moodService = moodService;
this.recommender = recommender;
this.recommendationService = recommendationService;
}
/// <summary>Save the model parameters of a recommender to a file</summary>
/// <remarks>
/// Does not save if filename is an empty string.
/// </remarks>
/// <param name="recommender">the recommender to store</param>
/// <param name="filename">the filename (may include relative paths)</param>
public static void Save(IRecommender recommender, string filename)
{
if (filename == null)
{
return;
}
Console.Error.WriteLine("Save model to {0}", filename);
recommender.SaveModel(filename);
}
/// <summary>Predict item scores for Track 2</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="candidates">a mapping from user IDs to the candidate items</param>
/// <param name="writer">the writer to write the scores to</param>
public static void PredictScoresTrack2(IRecommender recommender, Dictionary <int, IList <int> > candidates, BinaryWriter writer)
{
foreach (int user_id in candidates.Keys)
{
foreach (int item_id in candidates[user_id])
{
writer.Write(recommender.Predict(user_id, item_id));
}
}
}
/// <summary>Create a group recommender from the type name</summary>
/// <param name="typename">a string containing the type name</param>
/// <param name="recommender">the underlying recommender</param>
/// <returns>a group recommender object of type typename if the recommender type is found, null otherwise</returns>
public static GroupRecommender CreateGroupRecommender(this string typename, IRecommender recommender)
{
foreach (Assembly assembly in AppDomain.CurrentDomain.GetAssemblies())
{
Type type = assembly.GetType("MyMediaLite.GroupRecommendation." + typename, false, true);
if (type != null)
return CreateGroupRecommender(type, recommender);
}
return null;
}
/// <summary>Load the model parameters of a recommender from a file</summary>
/// <remarks>
/// Does not load model if filename is an empty string.
/// </remarks>
/// <param name="recommender">the <see cref="IRecommender"/> to save</param>
/// <param name="filename">the filename template</param>
public static void LoadModel(IRecommender recommender, string filename)
{
if (filename == string.Empty)
{
return;
}
Console.Error.WriteLine("Load model from {0}", filename);
recommender.LoadModel(filename);
}
/// <summary>Rate a given set of instances and write it to a TextWriter</summary>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">test cases</param>
/// <param name="writer">the TextWriter to write the predictions to</param>
/// <param name="user_mapping">an <see cref="Mapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="Mapping"/> object for the item IDs</param>
/// <param name="line_format">a format string specifying the line format; {0} is the user ID, {1} the item ID, {2} the rating</param>
/// <param name="header">if specified, write this string at the start of the output</param>
public static void WritePredictions(
this IRecommender recommender,
IRatings ratings,
TextWriter writer,
IMapping user_mapping = null,
IMapping item_mapping = null,
string line_format = "{0}\t{1}\t{2}",
string header = null)
{
if (user_mapping == null)
{
user_mapping = new IdentityMapping();
}
if (item_mapping == null)
{
item_mapping = new IdentityMapping();
}
if (header != null)
{
writer.WriteLine(header);
}
if (line_format == "ranking")
{
foreach (int user_id in ratings.AllUsers)
{
if (ratings.ByUser[user_id].Count > 0)
{
recommender.WritePredictions(
user_id,
new List <int>(from index in ratings.ByUser[user_id] select ratings.Items[index]),
new int[] { },
ratings.ByUser[user_id].Count,
writer,
user_mapping, item_mapping);
}
}
}
else
{
for (int index = 0; index < ratings.Count; index++)
{
writer.WriteLine(
string.Format(
CultureInfo.InvariantCulture,
line_format,
user_mapping.ToOriginalID(ratings.Users[index]),
item_mapping.ToOriginalID(ratings.Items[index]),
recommender.Predict(ratings.Users[index], ratings.Items[index])
)
);
}
}
}
public override void Setup()
{
try
{
// build recommender object
Type mmlRecommenderType = Helpers.ResolveType(SetupParameters["ml-class"]);
MmlRecommenderInstance = (IRecommender)mmlRecommenderType.GetConstructor(Type.EmptyTypes).Invoke(null);
if (typeof(ITimeAwareRatingPredictor).IsAssignableFrom(mmlRecommenderType))
{
DataType = DataType.TimeAwareRating;
}
else if (typeof(IRatingPredictor).IsAssignableFrom(mmlRecommenderType))
{
DataType = DataType.Ratings;
}
else if (typeof(ItemRecommender).IsAssignableFrom(mmlRecommenderType))
{
DataType = DataType.PosFeedback;
}
else
{
throw new WrapRecException(string.Format("Unknown MmlRecommender class: {0}", SetupParameters["ml-class"]));
}
// Set properties
foreach (var param in SetupParameters.Where(kv => kv.Key != "ml-class" && kv.Key != "numGroups" && kv.Key != "ingoreFeatures"))
{
PropertyInfo pi = MmlRecommenderInstance.GetType().GetProperty(param.Key);
// in case the value of attribute is empty ignore
// empty attributes are only used for logging purposes
if (string.IsNullOrEmpty(param.Value))
{
continue;
}
object paramVal;
if (pi.PropertyType.IsEnum)
{
paramVal = Enum.Parse(pi.PropertyType, param.Value);
}
else
{
paramVal = param.Value;
}
pi.SetValue(MmlRecommenderInstance, Convert.ChangeType(paramVal, pi.PropertyType));
}
}
catch (Exception ex)
{
throw new WrapRecException(string.Format("Cannot resolve MmlRecommender: {0}\n{1}", ex.Message, ex.StackTrace));
}
}
/// <summary>Rate a given set of instances and write it to a file</summary>
/// <param name="recommender">rating predictor</param>
/// <param name="ratings">test cases</param>
/// <param name="filename">the name of the file to write the predictions to</param>
/// <param name="user_mapping">an <see cref="Mapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="Mapping"/> object for the item IDs</param>
/// <param name="line_format">a format string specifying the line format; {0} is the user ID, {1} the item ID, {2} the rating</param>
/// <param name="header">if specified, write this string to the first line</param>
public static void WritePredictions(
this IRecommender recommender,
IRatings ratings,
string filename,
IMapping user_mapping = null, IMapping item_mapping = null,
string line_format = "{0}\t{1}\t{2}",
string header = null)
{
using (var writer = FileSystem.CreateStreamWriter(filename))
WritePredictions(recommender, ratings, writer, user_mapping, item_mapping, line_format);
}
/// <summary>Write item predictions (scores) for all users to a TextWriter object</summary>
/// <param name="recommender">the <see cref="IRecommender"/> to use for making the predictions</param>
/// <param name="train">a user-wise <see cref="IPosOnlyFeedback"/> containing the items already observed</param>
/// <param name="relevant_items">the list of candidate items</param>
/// <param name="num_predictions">the number of items to return per user, -1 if there should be no limit</param>
/// <param name="user_mapping">an <see cref="IEntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="IEntityMapping"/> object for the item IDs</param>
/// <param name="writer">the <see cref="TextWriter"/> to write to</param>
static public void WritePredictions(
IRecommender recommender,
IPosOnlyFeedback train,
ICollection <int> relevant_items,
int num_predictions,
IEntityMapping user_mapping, IEntityMapping item_mapping,
TextWriter writer)
{
var relevant_users = new List <int>(user_mapping.InternalIDs);
WritePredictions(recommender, train, relevant_users, relevant_items, num_predictions, user_mapping, item_mapping, writer);
}
public List <Suggestion> GetSuggest(UserBehavior db, long userId)
{
IRater rater = new SimpleRater();
IComparer comparer = new CorrelationUserComparer();
recommender = new ItemCollaborativeFilterRecommender(comparer, rater, 50);
recommender.Train(db);
var suggestion = recommender.GetSuggestions(userId, 500);
return(suggestion);
}
public static TestResults Test(this IRecommender classifier, UserBehaviorDatabase db, int numSuggestions)
{
// We're only using the ratings to check for existence of a rating, so we can use a simple rater for everything
SimpleRater rater = new SimpleRater();
UserBehaviorTransformer ubt = new UserBehaviorTransformer(db);
UserArticleRatingsTable ratings = ubt.GetUserArticleRatingsTable(rater);
int correctUsers = 0;
double averagePrecision = 0.0;
double averageRecall = 0.0;
// Get a list of users in this database who interacted with an article for the first time
List <int> distinctUsers = db.UserActions.Select(x => x.UserID).Distinct().ToList();
var distinctUserArticles = db.UserActions.GroupBy(x => new { x.UserID, x.ArticleID });
// Now get suggestions for each of these users
foreach (int user in distinctUsers)
{
List <Suggestion> suggestions = classifier.GetSuggestions(user, numSuggestions);
bool foundOne = false;
int userIndex = ratings.UserIndexToID.IndexOf(user);
int userCorrectArticles = 0;
int userTotalArticles = distinctUserArticles.Count(x => x.Key.UserID == user);
foreach (Suggestion s in suggestions)
{
int articleIndex = ratings.ArticleIndexToID.IndexOf(s.ArticleID);
// If one of the top N suggestions is what the user ended up reading, then we're golden
if (ratings.Users[userIndex].ArticleRatings[articleIndex] != 0)
{
userCorrectArticles++;
if (!foundOne)
{
correctUsers++;
foundOne = true;
}
}
}
averagePrecision += (double)userCorrectArticles / numSuggestions;
averageRecall += (double)userCorrectArticles / userTotalArticles;
}
averagePrecision /= distinctUsers.Count;
averageRecall /= distinctUsers.Count;
return(new TestResults(distinctUsers.Count, correctUsers, averageRecall, averagePrecision));
}
/// <summary>Write item predictions (scores) to a file</summary>
/// <param name="recommender">the <see cref="IRecommender"/> to use for making the predictions</param>
/// <param name="train">a user-wise <see cref="IPosOnlyFeedback"/> containing the items already observed</param>
/// <param name="candidate_items">list of candidate items</param>
/// <param name="num_predictions">number of items to return per user, -1 if there should be no limit</param>
/// <param name="filename">the name of the file to write to</param>
/// <param name="users">a list of users to make recommendations for</param>
/// <param name="user_mapping">an <see cref="IMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="IMapping"/> object for the item IDs</param>
/// <param name="repeated_items">true if items that a user has already accessed shall also be predicted</param>
static public void WritePredictions(
this IRecommender recommender,
IPosOnlyFeedback train,
IList <int> candidate_items,
int num_predictions,
string filename,
IList <int> users = null,
IMapping user_mapping = null, IMapping item_mapping = null,
bool repeated_items = false)
{
using (var writer = FileSystem.CreateStreamWriter(filename))
WritePredictions(recommender, train, candidate_items, num_predictions, writer, users, user_mapping, item_mapping, repeated_items);
}
public void testBestRating()
{
IRecommender recommender = buildRecommender();
IList <IRecommendedItem> recommended = recommender.Recommend(1, 1);
Assert.NotNull(recommended);
Assert.AreEqual(1, recommended.Count);
IRecommendedItem firstRecommended = recommended[0];
// item one should be recommended because it has a greater rating/score
Assert.AreEqual(2, firstRecommended.GetItemID());
Assert.AreEqual(0.1f, firstRecommended.GetValue(), EPSILON);
}
/// <summary>Sets a property of a MyMediaLite recommender</summary>
/// <param name="recommender">An <see cref="IRecommender"/></param>
/// <param name="key">the name of the property (case insensitive)</param>
/// <param name="val">the string representation of the value</param>
public static void SetProperty(IRecommender recommender, string key, string val)
{
Type type = recommender.GetType();
var property_names = new List <string>();
foreach (var p in type.GetProperties())
{
property_names.Add(p.Name);
}
property_names.Sort();
key = NormalizeName(key);
foreach (string property_name in property_names)
{
if (NormalizeName(property_name).StartsWith(key))
{
var property = type.GetProperty(property_name);
if (property.GetSetMethod() == null)
{
throw new ArgumentException(string.Format("Parameter '{0}' has no setter", key));
}
switch (property.PropertyType.ToString())
{
case "System.Double":
property.GetSetMethod().Invoke(recommender, new Object[] { double.Parse(val, CultureInfo.InvariantCulture) });
break;
case "System.Single":
property.GetSetMethod().Invoke(recommender, new Object[] { float.Parse(val, CultureInfo.InvariantCulture) });
break;
case "System.Int32":
property.GetSetMethod().Invoke(recommender, new Object[] { int.Parse(val) });
break;
case "System.UInt32":
property.GetSetMethod().Invoke(recommender, new Object[] { uint.Parse(val) });
break;
case "System.Boolean":
property.GetSetMethod().Invoke(recommender, new Object[] { bool.Parse(val) });
break;
default:
throw new ArgumentException(string.Format("Parameter '{0}' has unknown type '{1}'", key, property.PropertyType));
}
}
}
}
/// <summary>Predict items for a given user</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="user_id">the numerical ID of the user</param>
/// <param name="relevant_items">a collection of numerical IDs of relevant items</param>
/// <returns>an ordered list of items, the most likely item first</returns>
public static int[] PredictItems(IRecommender recommender, int user_id, ICollection<int> relevant_items)
{
var result = new List<WeightedItem>();
foreach (int item_id in relevant_items)
result.Add( new WeightedItem(item_id, recommender.Predict(user_id, item_id)));
result.Sort();
result.Reverse();
var return_array = new int[result.Count];
for (int i = 0; i < return_array.Length; i++)
return_array[i] = result[i].item_id;
return return_array;
}
/// <summary>predict items for a specific users</summary>
/// <param name="recommender">the <see cref="IRecommender"/> object to use for the predictions</param>
/// <param name="user_id">the user ID</param>
/// <param name="max_item_id">the maximum item ID</param>
/// <returns>a list sorted list of item IDs</returns>
public static int[] PredictItems(IRecommender recommender, int user_id, int max_item_id)
{
var result = new List<WeightedItem>();
for (int item_id = 0; item_id <= max_item_id; item_id++)
result.Add( new WeightedItem(item_id, recommender.Predict(user_id, item_id)));
result.Sort();
result.Reverse();
var return_array = new int[max_item_id + 1];
for (int i = 0; i < return_array.Length; i++)
return_array[i] = result[i].item_id;
return return_array;
}
/// <summary>Write item predictions (scores) to a TextWriter object</summary>
/// <param name="recommender">the <see cref="IRecommender"/> to use for making the predictions</param>
/// <param name="train">a user-wise <see cref="IPosOnlyFeedback"/> containing the items already observed</param>
/// <param name="relevant_users">a list of users to make recommendations for</param>
/// <param name="relevant_items">the list of candidate items</param>
/// <param name="num_predictions">the number of items to return per user, -1 if there should be no limit</param>
/// <param name="user_mapping">an <see cref="IEntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="IEntityMapping"/> object for the item IDs</param>
/// <param name="writer">the <see cref="TextWriter"/> to write to</param>
static public void WritePredictions(
IRecommender recommender,
IPosOnlyFeedback train,
IList <int> relevant_users,
ICollection <int> relevant_items,
int num_predictions,
IEntityMapping user_mapping, IEntityMapping item_mapping,
TextWriter writer)
{
foreach (int user_id in relevant_users)
{
var ignore_items = train.UserMatrix[user_id];
WritePredictions(recommender, user_id, relevant_items, ignore_items, num_predictions, user_mapping, item_mapping, writer);
}
}
public void testRecommender()
{
IRecommender recommender = buildRecommender();
IList <IRecommendedItem> recommended = recommender.Recommend(1, 1);
Assert.NotNull(recommended);
Assert.AreEqual(1, recommended.Count);
IRecommendedItem firstRecommended = recommended[0];
Assert.AreEqual(2, firstRecommended.GetItemID());
Assert.AreEqual(0.1f, firstRecommended.GetValue(), EPSILON);
recommender.Refresh(null);
Assert.AreEqual(2, firstRecommended.GetItemID());
Assert.AreEqual(0.1f, firstRecommended.GetValue(), EPSILON);
}
public static RecommenderResult RecommendUser(
this IRecommender recommender,
string userId)
{
if (recommender == null)
{
throw new ArgumentNullException(nameof(recommender));
}
if (userId == null)
{
throw new ArgumentNullException(nameof(userId));
}
return(recommender.RecommendUser(userId, RecommenderResultBuilderFactory.Instance));
}
public CachingRecommender(IRecommender recommender)
{
//Preconditions.checkArgument(recommender != null, "recommender is null");
this.recommender = recommender;
maxHowMany = new int[] { 1 };
// Use "num users" as an upper limit on cache size. Rough guess.
int numUsers = recommender.GetDataModel().GetNumUsers();
recommendationsRetriever = new RecommendationRetriever(this);
recommendationCache = new Cache <long, Recommendations>(recommendationsRetriever, numUsers);
estimatedPrefCache = new Cache <Tuple <long, long>, float>(new EstimatedPrefRetriever(this), numUsers);
refreshHelper = new RefreshHelper(() => {
clear();
});
refreshHelper.AddDependency(recommender);
}
public NewComputerDialogComponent(string dialogId, IPersonStateManager accessors, IDbContext dbContext,
IRecommender recommender,
string menuItemOptionText = null)
: base(dialogId)
{
_dbContext = dbContext;
_accessors = accessors;
_recommender = recommender;
MenuItemOptionText = menuItemOptionText ?? dialogId;
AddDialog(new WaterfallDialog(MAIN_WATERFALL)
.AddStep(DecideIfNewOrExistUser.Step(_dbContext, NEW_USER_DIALOG))
.AddStep(GetComputerRecommendations));
AddDialog(new PersonalDataDialogComponent(NEW_USER_DIALOG, _accessors, _dbContext));
}
/// <summary>Create a group recommender from a type object</summary>
/// <param name="type">the type object</param>
/// <param name="recommender">the underlying recommender</param>
/// <returns>a group recommender object of type type</returns>
public static GroupRecommender CreateGroupRecommender(this Type type, IRecommender recommender)
{
if (type.IsAbstract)
return null;
if (type.IsGenericType)
return null;
if (recommender == null)
throw new ArgumentNullException("recommender");
if (type == null)
throw new ArgumentNullException("type");
if (type.IsSubclassOf(typeof(GroupRecommender)))
return (GroupRecommender) type.GetConstructor(new Type[] { } ).Invoke( new object[] { recommender });
else
throw new Exception(type.Name + " is not a subclass of MyMediaLite.GroupRecommendation.GroupRecommender");
}
public static LoadStatistics runLoad(IRecommender recommender, int howMany) {
IDataModel dataModel = recommender.GetDataModel();
int numUsers = dataModel.GetNumUsers();
double sampleRate = 1000.0 / numUsers;
var userSampler =
SamplinglongPrimitiveIterator.MaybeWrapIterator(dataModel.GetUserIDs(), sampleRate);
if (userSampler.MoveNext())
recommender.Recommend(userSampler.Current, howMany); // Warm up
var callables = new List<Action>();
while (userSampler.MoveNext()) {
callables.Add(new LoadCallable(recommender, userSampler.Current).call);
}
AtomicInteger noEstimateCounter = new AtomicInteger();
IRunningAverageAndStdDev timing = new FullRunningAverageAndStdDev();
AbstractDifferenceRecommenderEvaluator.execute(callables, noEstimateCounter, timing);
return new LoadStatistics(timing);
}
/// <summary>Constructor that takes the underlying recommender that will be used</summary>
/// <param name="recommender">the underlying recommender</param>
public GroupRecommender(IRecommender recommender)
{
this.recommender = recommender;
}
///
public Average(IRecommender recommender)
: base(recommender)
{
}
private double getEvaluation(FastByIDMap<IPreferenceArray> testPrefs, IRecommender recommender)
{
reset();
var estimateCallables = new List<Action>();
AtomicInteger noEstimateCounter = new AtomicInteger();
foreach (var entry in testPrefs.EntrySet()) {
estimateCallables.Add( () => {
var testUserID = entry.Key;
var prefs = entry.Value;
foreach (IPreference realPref in prefs) {
float estimatedPreference = float.NaN;
try {
estimatedPreference = recommender.EstimatePreference(testUserID, realPref.GetItemID());
} catch (NoSuchUserException nsue) {
// It's possible that an item exists in the test data but not training data in which case
// NSEE will be thrown. Just ignore it and move on.
log.Info("User exists in test data but not training data: {}", testUserID);
} catch (NoSuchItemException nsie) {
log.Info("Item exists in test data but not training data: {}", realPref.GetItemID());
}
if (float.IsNaN(estimatedPreference)) {
noEstimateCounter.incrementAndGet();
} else {
estimatedPreference = capEstimatedPreference(estimatedPreference);
processOneEstimate(estimatedPreference, realPref);
}
}
});
// new PreferenceEstimateCallable(recommender, entry.Key, entry.Value, noEstimateCounter));
}
log.Info("Beginning evaluation of {} users", estimateCallables.Count);
IRunningAverageAndStdDev timing = new FullRunningAverageAndStdDev();
execute(estimateCallables, noEstimateCounter, timing);
return computeFinalEvaluation();
}
/// <summary>Predict ratings for Track 1</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="ratings">the ratings to predict</param>
/// <param name="filename">the file to write the predictions to</param>
public static void PredictRatings(IRecommender recommender, IRatings ratings, string filename)
{
using (var stream = new FileStream(filename, FileMode.Create))
using (var writer = new BinaryWriter(stream))
PredictRatings(recommender, ratings, writer);
}
/// <summary>Predict items for Track 2</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="candidates">a mapping from user IDs to the candidate items</param>
/// <param name="writer">the writer object to write the predictions to</param>
public static void PredictTrack2(IRecommender recommender, Dictionary<int, IList<int>> candidates, TextWriter writer)
{
foreach (int user_id in candidates.Keys) // this is ordered, but is it guaranteed?
{
IList<int> user_candidates = candidates[user_id];
var predictions = new double[user_candidates.Count];
for (int i = 0; i < user_candidates.Count; i++)
predictions[i] = recommender.Predict(user_id, user_candidates[i]);
var positions = new List<int>(new int[] { 0, 1, 2, 3, 4, 5 });
positions.Sort(delegate(int pos1, int pos2) { return predictions[pos2].CompareTo(predictions[pos1]); } );
for (int i = 0; i < user_candidates.Count; i++)
if (positions.IndexOf(i) < 3)
writer.Write("1");
else
writer.Write("0");
}
}
///
public WeightedAverage(IRecommender recommender)
: base(recommender)
{
}
/// <summary>Load the model parameters of a recommender from a file</summary>
/// <param name="recommender">the <see cref="IRecommender"/> to load</param>
/// <param name="filename">the filename template</param>
public static void Load(IRecommender recommender, string filename)
{
Console.Error.WriteLine("Load model from {0}", filename);
recommender.LoadModel(filename);
}
// TODO there are too many different versions of this method interface - we should simplify the API
/// <summary>Write item predictions (scores) for all users to a file</summary>
/// <param name="recommender">the <see cref="IRecommender"/> to use for making the predictions</param>
/// <param name="train">a user-wise <see cref="IPosOnlyFeedback"/> containing the items already observed</param>
/// <param name="relevant_items">the list of candidate items</param>
/// <param name="num_predictions">the number of items to return per user, -1 if there should be no limit</param>
/// <param name="user_mapping">an <see cref="IEntityMapping"/> object for the user IDs</param>
/// <param name="item_mapping">an <see cref="IEntityMapping"/> object for the item IDs</param>
/// <param name="filename">the name of the file to write to</param>
public static void WritePredictions(
IRecommender recommender,
IPosOnlyFeedback train,
ICollection<int> relevant_items,
int num_predictions,
IEntityMapping user_mapping, IEntityMapping item_mapping,
string filename)
{
if (filename.Equals("-"))
WritePredictions(recommender, train, relevant_items, num_predictions, user_mapping, item_mapping, Console.Out);
else
using ( var writer = new StreamWriter(filename) )
WritePredictions(recommender, train, relevant_items, num_predictions, user_mapping, item_mapping, writer);
}
public MedianRatingRecommendationSystem(ITrainer<IMedianRatingModel> trainer, IRecommender<IMedianRatingModel> recommender)
{
Trainer = trainer;
Recommender = recommender;
}
///
public PairwiseWins(IRecommender recommender)
: base(recommender)
{
}
public SimpleKnnRecommendationSystem(IRecommender<ISimpleKnnModel> recommender)
: this(new SimpleKnnTrainer(), recommender)
{
}
public static LoadStatistics runLoad(IRecommender recommender) {
return runLoad(recommender, 10);
}
/// <summary>Predict item scores for Track 2</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="candidates">a mapping from user IDs to the candidate items</param>
/// <param name="writer">the writer to write the scores to</param>
public static void PredictScoresTrack2(IRecommender recommender, Dictionary<int, IList<int>> candidates, BinaryWriter writer)
{
foreach (int user_id in candidates.Keys)
foreach (int item_id in candidates[user_id])
writer.Write(recommender.Predict(user_id, item_id));
}
public AverageRatingRecommendationSystem(ITrainer<IAverageRatingModel> trainer, IRecommender<IAverageRatingModel> recommender)
{
Trainer = trainer;
Recommender = recommender;
}
/// <summary>Predict items for Track 2</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="candidates">a mapping from user IDs to the candidate items</param>
/// <param name="filename">the file to write the predictions to</param>
public static void PredictTrack2(IRecommender recommender, Dictionary<int, IList<int>> candidates, string filename)
{
using (FileStream file_stream = File.Create(filename))
using (var writer = new StreamWriter(file_stream))
PredictTrack2(recommender, candidates, writer);
}
public SimpleKnnRecommendationSystem(ITrainer<ISimpleKnnModel> trainer, IRecommender<ISimpleKnnModel> recommender)
{
Trainer = trainer;
Recommender = recommender;
}
/// <summary>Predict ratings (double precision)</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="ratings">the ratings to predict</param>
/// <param name="writer">the writer object to write the predictions to</param>
public static void PredictRatingsDouble(IRecommender recommender, IRatings ratings, BinaryWriter writer)
{
for (int i = 0; i < ratings.Count; i++)
writer.Write(recommender.Predict(ratings.Users[i], ratings.Items[i]).ToString(CultureInfo.InvariantCulture));
}
public LoadCallable(IRecommender recommender, long userID) {
this.recommender = recommender;
this.userID = userID;
}
/// <summary>Predict ratings for Track 1</summary>
/// <param name="recommender">the recommender to use</param>
/// <param name="ratings">the ratings to predict</param>
/// <param name="writer">the writer object to write the predictions to</param>
public static void PredictRatings(IRecommender recommender, IRatings ratings, BinaryWriter writer)
{
for (int i = 0; i < ratings.Count; i++)
{
double prediction = recommender.Predict(ratings.Users[i], ratings.Items[i]);
byte encoded_prediction = (byte) (2.55 * prediction + 0.5);
writer.Write(encoded_prediction);
}
}
///
public Minimum(IRecommender recommender)
: base(recommender)
{
}
public static void Main(string[] args)
{
AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(MyMediaLite.Util.Handlers.UnhandledExceptionHandler);
Console.CancelKeyPress += new ConsoleCancelEventHandler(AbortHandler);
// recommender arguments
string method = null;
string recommender_options = string.Empty;
// help/version
bool show_help = false;
bool show_version = false;
// variables for iteration search
int max_iter = 500;
double cutoff = 0;
double epsilon = 0;
string measure = "AUC";
compute_fit = false;
// other parameters
test_ratio = 0;
num_test_users = -1;
repeat_eval = false;
var p = new OptionSet() {
// string-valued options
{ "training-file=", v => training_file = v },
{ "test-file=", v => test_file = v },
{ "recommender=", v => method = v },
{ "group-recommender=", v => group_method = v },
{ "recommender-options=", v => recommender_options += " " + v },
{ "data-dir=", v => data_dir = v },
{ "user-attributes=", v => user_attributes_file = v },
{ "item-attributes=", v => item_attributes_file = v },
{ "user-relations=", v => user_relations_file = v },
{ "item-relations=", v => item_relations_file = v },
{ "save-model=", v => save_model_file = v },
{ "load-model=", v => load_model_file = v },
{ "save-user-mapping=", v => save_user_mapping_file = v },
{ "save-item-mapping=", v => save_item_mapping_file = v },
{ "load-user-mapping=", v => load_user_mapping_file = v },
{ "load-item-mapping=", v => load_item_mapping_file = v },
{ "prediction-file=", v => prediction_file = v },
{ "test-users=", v => test_users_file = v },
{ "candidate-items=", v => candidate_items_file = v },
{ "user-groups=", v => user_groups_file = v },
{ "measure=", v => measure = v },
// integer-valued options
{ "find-iter=", (int v) => find_iter = v },
{ "max-iter=", (int v) => max_iter = v },
{ "random-seed=", (int v) => random_seed = v },
{ "predict-items-number=", (int v) => predict_items_number = v },
{ "num-test-users=", (int v) => num_test_users = v },
{ "cross-validation=", (uint v) => cross_validation = v },
// floating point options
{ "epsilon=", (double v) => epsilon = v },
{ "cutoff=", (double v) => cutoff = v },
{ "test-ratio=", (double v) => test_ratio = v },
{ "rating-threshold=", (float v) => rating_threshold = v },
// enum options
{ "file-format=", (ItemDataFileFormat v) => file_format = v },
// boolean options
{ "user-prediction", v => user_prediction = v != null },
{ "compute-fit", v => compute_fit = v != null },
{ "online-evaluation", v => online_eval = v != null },
{ "repeat-evaluation", v => repeat_eval = v != null },
{ "no-id-mapping", v => no_id_mapping = v != null },
{ "overlap-items", v => overlap_items = v != null },
{ "all-items", v => all_items = v != null },
{ "in-training-items", v => in_training_items = v != null },
{ "in-test-items", v => in_test_items = v != null },
{ "help", v => show_help = v != null },
{ "version", v => show_version = v != null },
};
IList<string> extra_args = p.Parse(args);
bool no_eval = true;
if (test_ratio > 0 || test_file != null)
no_eval = false;
if (show_version)
ShowVersion();
if (show_help)
Usage(0);
if (random_seed != -1)
MyMediaLite.Util.Random.Seed = random_seed;
// set up recommender
if (load_model_file != null)
recommender = Model.Load(load_model_file);
else if (method != null)
recommender = Recommender.CreateItemRecommender(method);
else
recommender = Recommender.CreateItemRecommender("MostPopular");
// in case something went wrong ...
if (recommender == null && method != null)
Usage(string.Format("Unknown recommendation method: '{0}'", method));
if (recommender == null && load_model_file != null)
Abort(string.Format("Could not load model from file {0}.", load_model_file));
CheckParameters(extra_args);
recommender.Configure(recommender_options, (string m) => { Console.Error.WriteLine(m); Environment.Exit(-1); });
if (no_id_mapping)
{
user_mapping = new IdentityMapping();
item_mapping = new IdentityMapping();
}
if (load_user_mapping_file != null)
user_mapping = EntityMappingExtensions.LoadMapping(load_user_mapping_file);
if (load_item_mapping_file != null)
item_mapping = EntityMappingExtensions.LoadMapping(load_item_mapping_file);
// load all the data
LoadData();
Console.Write(training_data.Statistics(test_data, user_attributes, item_attributes));
// if requested, save ID mappings
if (save_user_mapping_file != null)
user_mapping.SaveMapping(save_user_mapping_file);
if (save_item_mapping_file != null)
item_mapping.SaveMapping(save_item_mapping_file);
TimeSpan time_span;
if (find_iter != 0)
{
if ( !(recommender is IIterativeModel) )
Abort("Only iterative recommenders (interface IIterativeModel) support --find-iter=N.");
var iterative_recommender = (IIterativeModel) recommender;
Console.WriteLine(recommender);
var eval_stats = new List<double>();
if (cross_validation > 1)
{
recommender.DoIterativeCrossValidation(cross_validation, test_users, candidate_items, eval_item_mode, repeat_eval, max_iter, find_iter);
}
else
{
if (load_model_file == null)
recommender.Train();
if (compute_fit)
Console.WriteLine("fit: {0} iteration {1} ", ComputeFit(), iterative_recommender.NumIter);
var results = Evaluate();
Console.WriteLine("{0} iteration {1}", results, iterative_recommender.NumIter);
for (int it = (int) iterative_recommender.NumIter + 1; it <= max_iter; it++)
{
TimeSpan t = Wrap.MeasureTime(delegate() {
iterative_recommender.Iterate();
});
training_time_stats.Add(t.TotalSeconds);
if (it % find_iter == 0)
{
if (compute_fit)
{
t = Wrap.MeasureTime(delegate() {
Console.WriteLine("fit: {0} iteration {1} ", ComputeFit(), it);
});
fit_time_stats.Add(t.TotalSeconds);
}
t = Wrap.MeasureTime(delegate() { results = Evaluate(); });
eval_time_stats.Add(t.TotalSeconds);
eval_stats.Add(results[measure]);
Console.WriteLine("{0} iteration {1}", results, it);
Model.Save(recommender, save_model_file, it);
Predict(prediction_file, test_users_file, it);
if (epsilon > 0.0 && eval_stats.Max() - results[measure] > epsilon)
{
Console.Error.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0} >> {1}", results["RMSE"], eval_stats.Min()));
Console.Error.WriteLine("Reached convergence on training/validation data after {0} iterations.", it);
break;
}
if (results[measure] < cutoff)
{
Console.Error.WriteLine("Reached cutoff after {0} iterations.", it);
Console.Error.WriteLine("DONE");
break;
}
}
} // for
}
}
else
{
Console.WriteLine(recommender + " ");
if (load_model_file == null)
{
if (cross_validation > 1)
{
var results = recommender.DoCrossValidation(cross_validation, test_users, candidate_items, eval_item_mode, compute_fit, true);
Console.Write(results);
no_eval = true;
}
else
{
time_span = Wrap.MeasureTime( delegate() { recommender.Train(); } );
Console.Write("training_time " + time_span + " ");
}
}
if (prediction_file != null)
{
Predict(prediction_file, test_users_file);
}
else if (!no_eval)
{
if (compute_fit)
Console.WriteLine("fit: {0}", ComputeFit());
if (online_eval)
time_span = Wrap.MeasureTime( delegate() {
var results = recommender.EvaluateOnline(test_data, training_data, test_users, candidate_items, eval_item_mode);
Console.Write(results);
});
else if (group_method != null)
{
GroupRecommender group_recommender = null;
Console.Write("group recommendation strategy: {0} ", group_method);
// TODO GroupUtils.CreateGroupRecommender(group_method, recommender);
if (group_method == "Average")
group_recommender = new Average(recommender);
else if (group_method == "Minimum")
group_recommender = new Minimum(recommender);
else if (group_method == "Maximum")
group_recommender = new Maximum(recommender);
else
Usage("Unknown group recommendation strategy in --group-recommender=METHOD");
time_span = Wrap.MeasureTime( delegate() {
var result = group_recommender.Evaluate(test_data, training_data, group_to_user, candidate_items);
Console.Write(result);
});
}
else
time_span = Wrap.MeasureTime( delegate() { Console.Write(Evaluate()); });
Console.Write(" testing_time " + time_span);
}
Console.WriteLine();
}
Model.Save(recommender, save_model_file);
DisplayStats();
}
