public virtual double evaluate(RecommenderBuilder recommenderBuilder, DataModelBuilder dataModelBuilder, DataModel dataModel, double trainingPercentage, double evaluationPercentage) { log.info("Beginning evaluation using {} of {}", new object[] { trainingPercentage, dataModel }); int num = dataModel.getNumUsers(); FastByIDMap <PreferenceArray> trainingPrefs = new FastByIDMap <PreferenceArray>(1 + ((int)(evaluationPercentage * num))); FastByIDMap <PreferenceArray> testPrefs = new FastByIDMap <PreferenceArray>(1 + ((int)(evaluationPercentage * num))); IEnumerator <long> enumerator = dataModel.getUserIDs(); while (enumerator.MoveNext()) { long current = enumerator.Current; if (this.random.nextDouble() < evaluationPercentage) { this.splitOneUsersPrefs(trainingPercentage, trainingPrefs, testPrefs, current, dataModel); } } DataModel model = (dataModelBuilder == null) ? new GenericDataModel(trainingPrefs) : dataModelBuilder.buildDataModel(trainingPrefs); Recommender recommender = recommenderBuilder.buildRecommender(model); double num3 = this.getEvaluation(testPrefs, recommender); log.info("Evaluation result: {}", new object[] { num3 }); return(num3); }
/** * {@inheritDoc} */ public double Evaluate(RecommenderBuilder recommenderBuilder, DataModel dataModel, double trainingPercentage, double evaluationPercentage) { if (recommenderBuilder == null) { throw new ArgumentNullException("recommenderBuilder is null"); } if (dataModel == null) { throw new ArgumentNullException("dataModel is null"); } if (double.IsNaN(trainingPercentage) || trainingPercentage <= 0.0 || trainingPercentage >= 1.0) { throw new ArgumentException("Invalid trainingPercentage: " + trainingPercentage); } if (double.IsNaN(evaluationPercentage) || evaluationPercentage <= 0.0 || evaluationPercentage > 1.0) { throw new ArgumentException("Invalid evaluationPercentage: " + evaluationPercentage); } log.Info("Beginning evaluation using " + trainingPercentage + " of " + dataModel); int numUsers = dataModel.GetNumUsers(); ICollection<User> trainingUsers = new List<User>(1 + (int) (trainingPercentage * (double) numUsers)); IDictionary<User, ICollection<Preference>> testUserPrefs = new Dictionary<User, ICollection<Preference>>(1 + (int) ((1.0 - trainingPercentage) * (double) numUsers)); foreach (User user in dataModel.GetUsers()) { if (random.NextDouble() < evaluationPercentage) { ICollection<Preference> trainingPrefs = new List<Preference>(); ICollection<Preference> testPrefs = new List<Preference>(); Preference[] prefs = user.GetPreferencesAsArray(); foreach (Preference pref in prefs) { Item itemCopy = new GenericItem<String>(pref.Item.ID.ToString()); Preference newPref = new GenericPreference(null, itemCopy, pref.Value); if (random.NextDouble() < trainingPercentage) { trainingPrefs.Add(newPref); } else { testPrefs.Add(newPref); } } if (log.IsDebugEnabled) { log.Debug("Training against " + trainingPrefs.Count + " preferences"); log.Debug("Evaluating accuracy of " + testPrefs.Count + " preferences"); } if (trainingPrefs.Count > 0) { User trainingUser = new GenericUser<String>(user.ID.ToString(), trainingPrefs); trainingUsers.Add(trainingUser); if (testPrefs.Count > 0) { testUserPrefs.Add(trainingUser, testPrefs); } } } } DataModel trainingModel = new GenericDataModel(trainingUsers); Recommender recommender = recommenderBuilder.BuildRecommender(trainingModel); double result = GetEvaluation(testUserPrefs, recommender); log.Info("Evaluation result: " + result); return result; }
public IRStatistics Evaluate(RecommenderBuilder recommenderBuilder, DataModel dataModel, int at, double relevanceThreshold, double evaluationPercentage) { if (recommenderBuilder == null) { throw new ArgumentNullException("recommenderBuilder is null"); } if (dataModel == null) { throw new ArgumentNullException("dataModel is null"); } if (at < 1) { throw new ArgumentException("at must be at least 1"); } if (double.IsNaN(evaluationPercentage) || evaluationPercentage <= 0.0 || evaluationPercentage > 1.0) { throw new ArgumentException("Invalid evaluationPercentage: " + evaluationPercentage); } if (double.IsNaN(relevanceThreshold)) { throw new ArgumentException("Invalid relevanceThreshold: " + evaluationPercentage); } RunningAverage precision = new FullRunningAverage(); RunningAverage recall = new FullRunningAverage(); foreach (User user in dataModel.GetUsers()) { Object id = user.ID; if (random.NextDouble() < evaluationPercentage) { ICollection <Item> relevantItems = new HashedSet <Item>(/* at */); Preference[] prefs = user.GetPreferencesAsArray(); foreach (Preference pref in prefs) { if (pref.Value >= relevanceThreshold) { relevantItems.Add(pref.Item); } } int numRelevantItems = relevantItems.Count; if (numRelevantItems > 0) { ICollection <User> trainingUsers = new List <User>(dataModel.GetNumUsers()); foreach (User user2 in dataModel.GetUsers()) { if (id.Equals(user2.ID)) { ICollection <Preference> trainingPrefs = new List <Preference>(); prefs = user2.GetPreferencesAsArray(); foreach (Preference pref in prefs) { if (!relevantItems.Contains(pref.Item)) { trainingPrefs.Add(pref); } } if (trainingPrefs.Count > 0) { User trainingUser = new GenericUser <String>(id.ToString(), trainingPrefs); trainingUsers.Add(trainingUser); } } else { trainingUsers.Add(user2); } } DataModel trainingModel = new GenericDataModel(trainingUsers); Recommender recommender = recommenderBuilder.BuildRecommender(trainingModel); try { trainingModel.GetUser(id); } catch (NoSuchElementException) { continue; // Oops we excluded all prefs for the user -- just move on } int intersectionSize = 0; foreach (RecommendedItem recommendedItem in recommender.Recommend(id, at)) { if (relevantItems.Contains(recommendedItem.Item)) { intersectionSize++; } } precision.AddDatum((double)intersectionSize / (double)at); recall.AddDatum((double)intersectionSize / (double)numRelevantItems); } } } return(new IRStatisticsImpl(precision.Average, recall.Average)); }
public IRStatistics Evaluate(RecommenderBuilder recommenderBuilder, DataModel dataModel, int at, double relevanceThreshold, double evaluationPercentage) { if (recommenderBuilder == null) { throw new ArgumentNullException("recommenderBuilder is null"); } if (dataModel == null) { throw new ArgumentNullException("dataModel is null"); } if (at < 1) { throw new ArgumentException("at must be at least 1"); } if (double.IsNaN(evaluationPercentage) || evaluationPercentage <= 0.0 || evaluationPercentage > 1.0) { throw new ArgumentException("Invalid evaluationPercentage: " + evaluationPercentage); } if (double.IsNaN(relevanceThreshold)) { throw new ArgumentException("Invalid relevanceThreshold: " + evaluationPercentage); } RunningAverage precision = new FullRunningAverage(); RunningAverage recall = new FullRunningAverage(); foreach (User user in dataModel.GetUsers()) { Object id = user.ID; if (random.NextDouble() < evaluationPercentage) { ICollection<Item> relevantItems = new HashedSet<Item>(/* at */); Preference[] prefs = user.GetPreferencesAsArray(); foreach (Preference pref in prefs) { if (pref.Value >= relevanceThreshold) { relevantItems.Add(pref.Item); } } int numRelevantItems = relevantItems.Count; if (numRelevantItems > 0) { ICollection<User> trainingUsers = new List<User>(dataModel.GetNumUsers()); foreach (User user2 in dataModel.GetUsers()) { if (id.Equals(user2.ID)) { ICollection<Preference> trainingPrefs = new List<Preference>(); prefs = user2.GetPreferencesAsArray(); foreach (Preference pref in prefs) { if (!relevantItems.Contains(pref.Item)) { trainingPrefs.Add(pref); } } if (trainingPrefs.Count > 0) { User trainingUser = new GenericUser<String>(id.ToString(), trainingPrefs); trainingUsers.Add(trainingUser); } } else { trainingUsers.Add(user2); } } DataModel trainingModel = new GenericDataModel(trainingUsers); Recommender recommender = recommenderBuilder.BuildRecommender(trainingModel); try { trainingModel.GetUser(id); } catch (NoSuchElementException) { continue; // Oops we excluded all prefs for the user -- just move on } int intersectionSize = 0; foreach (RecommendedItem recommendedItem in recommender.Recommend(id, at)) { if (relevantItems.Contains(recommendedItem.Item)) { intersectionSize++; } } precision.AddDatum((double) intersectionSize / (double) at); recall.AddDatum((double) intersectionSize / (double) numRelevantItems); } } } return new IRStatisticsImpl(precision.Average, recall.Average); }
public IRStatistics evaluate(RecommenderBuilder recommenderBuilder, DataModelBuilder dataModelBuilder, DataModel dataModel, IDRescorer rescorer, int at, double relevanceThreshold, double evaluationPercentage) { int num = dataModel.getNumItems(); RunningAverage average = new FullRunningAverage(); RunningAverage average2 = new FullRunningAverage(); RunningAverage average3 = new FullRunningAverage(); RunningAverage average4 = new FullRunningAverage(); int num2 = 0; int num3 = 0; IEnumerator <long> enumerator = dataModel.getUserIDs(); while (enumerator.MoveNext()) { long current = enumerator.Current; if (this.random.nextDouble() < evaluationPercentage) { Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); PreferenceArray prefs = dataModel.getPreferencesFromUser(current); double num5 = double.IsNaN(relevanceThreshold) ? computeThreshold(prefs) : relevanceThreshold; FastIDSet relevantItemIDs = this.dataSplitter.getRelevantItemsIDs(current, at, num5, dataModel); int num6 = relevantItemIDs.size(); if (num6 > 0) { FastByIDMap <PreferenceArray> trainingUsers = new FastByIDMap <PreferenceArray>(dataModel.getNumUsers()); IEnumerator <long> enumerator2 = dataModel.getUserIDs(); while (enumerator2.MoveNext()) { this.dataSplitter.processOtherUser(current, relevantItemIDs, trainingUsers, enumerator2.Current, dataModel); } DataModel model = (dataModelBuilder == null) ? new GenericDataModel(trainingUsers) : dataModelBuilder.buildDataModel(trainingUsers); try { model.getPreferencesFromUser(current); } catch (NoSuchUserException) { continue; } int num7 = num6 + model.getItemIDsFromUser(current).size(); if (num7 >= (2 * at)) { Recommender recommender = recommenderBuilder.buildRecommender(model); int num8 = 0; List <RecommendedItem> list = recommender.recommend(current, at, rescorer); foreach (RecommendedItem item in list) { if (relevantItemIDs.contains(item.getItemID())) { num8++; } } int count = list.Count; if (count > 0) { average.addDatum(((double)num8) / ((double)count)); } average2.addDatum(((double)num8) / ((double)num6)); if (num6 < num7) { average3.addDatum(((double)(count - num8)) / ((double)(num - num6))); } double num10 = 0.0; double num11 = 0.0; for (int i = 0; i < count; i++) { RecommendedItem item2 = list[i]; double num13 = 1.0 / log2(i + 2.0); if (relevantItemIDs.contains(item2.getItemID())) { num10 += num13; } if (i < num6) { num11 += num13; } } if (num11 > 0.0) { average4.addDatum(num10 / num11); } num2++; if (count > 0) { num3++; } stopwatch.Stop(); log.info("Evaluated with user {} in {}ms", new object[] { current, stopwatch.ElapsedMilliseconds }); log.info("Precision/recall/fall-out/nDCG/reach: {} / {} / {} / {} / {}", new object[] { average.getAverage(), average2.getAverage(), average3.getAverage(), average4.getAverage(), ((double)num3) / ((double)num2) }); } } } } return(new IRStatisticsImpl(average.getAverage(), average2.getAverage(), average3.getAverage(), average4.getAverage(), ((double)num3) / ((double)num2))); }
/** * {@inheritDoc} */ public double Evaluate(RecommenderBuilder recommenderBuilder, DataModel dataModel, double trainingPercentage, double evaluationPercentage) { if (recommenderBuilder == null) { throw new ArgumentNullException("recommenderBuilder is null"); } if (dataModel == null) { throw new ArgumentNullException("dataModel is null"); } if (double.IsNaN(trainingPercentage) || trainingPercentage <= 0.0 || trainingPercentage >= 1.0) { throw new ArgumentException("Invalid trainingPercentage: " + trainingPercentage); } if (double.IsNaN(evaluationPercentage) || evaluationPercentage <= 0.0 || evaluationPercentage > 1.0) { throw new ArgumentException("Invalid evaluationPercentage: " + evaluationPercentage); } log.Info("Beginning evaluation using " + trainingPercentage + " of " + dataModel); int numUsers = dataModel.GetNumUsers(); ICollection <User> trainingUsers = new List <User>(1 + (int)(trainingPercentage * (double)numUsers)); IDictionary <User, ICollection <Preference> > testUserPrefs = new Dictionary <User, ICollection <Preference> >(1 + (int)((1.0 - trainingPercentage) * (double)numUsers)); foreach (User user in dataModel.GetUsers()) { if (random.NextDouble() < evaluationPercentage) { ICollection <Preference> trainingPrefs = new List <Preference>(); ICollection <Preference> testPrefs = new List <Preference>(); Preference[] prefs = user.GetPreferencesAsArray(); foreach (Preference pref in prefs) { Item itemCopy = new GenericItem <String>(pref.Item.ID.ToString()); Preference newPref = new GenericPreference(null, itemCopy, pref.Value); if (random.NextDouble() < trainingPercentage) { trainingPrefs.Add(newPref); } else { testPrefs.Add(newPref); } } if (log.IsDebugEnabled) { log.Debug("Training against " + trainingPrefs.Count + " preferences"); log.Debug("Evaluating accuracy of " + testPrefs.Count + " preferences"); } if (trainingPrefs.Count > 0) { User trainingUser = new GenericUser <String>(user.ID.ToString(), trainingPrefs); trainingUsers.Add(trainingUser); if (testPrefs.Count > 0) { testUserPrefs.Add(trainingUser, testPrefs); } } } } DataModel trainingModel = new GenericDataModel(trainingUsers); Recommender recommender = recommenderBuilder.BuildRecommender(trainingModel); double result = GetEvaluation(testUserPrefs, recommender); log.Info("Evaluation result: " + result); return(result); }