public IList <RecommendedItem> MostSimilarItems(List <Object> itemIDs,
int howMany,
Rescorer <Pair <Item, Item> > rescorer)
{
if (rescorer == null)
{
throw new ArgumentNullException("rescorer is null");
}
DataModel model = this.DataModel;
List <Item> toItems = new List <Item>(itemIDs.Count);
foreach (Object itemID in itemIDs)
{
toItems.Add(model.GetItem(itemID));
}
TopItems.Estimator <Item> estimator = new MultiMostSimilarEstimator(toItems, correlation, rescorer);
ICollection <Item> allItems = new HashedSet <Item>(/*Model.GetNumItems()*/);
foreach (Item item in model.GetItems())
{
allItems.Add(item);
}
foreach (Item item in toItems)
{
allItems.Remove(item);
}
return(TopItems.GetTopItems(howMany, allItems, NullRescorer <Item> .Instance, estimator));
}
public virtual void TestBasic()
{
// create a sort field and sort by it (reverse order)
Query query = new TermQuery(new Term("body", "contents"));
IndexReader r = searcher.IndexReader;
// Just first pass query
TopDocs hits = searcher.Search(query, 10);
AreEqual(3, hits.TotalHits);
AreEqual("3", r.Document(hits.ScoreDocs[0].Doc).Get("id"));
AreEqual("1", r.Document(hits.ScoreDocs[1].Doc).Get("id"));
AreEqual("2", r.Document(hits.ScoreDocs[2].Doc).Get("id"));
// Now, rescore:
Expression e = JavascriptCompiler.Compile("sqrt(_score) + ln(popularity)");
SimpleBindings bindings = new SimpleBindings();
bindings.Add(new SortField("popularity", SortField.Type_e.INT));
bindings.Add(new SortField("_score", SortField.Type_e.SCORE));
Rescorer rescorer = e.GetRescorer(bindings);
hits = rescorer.Rescore(searcher, hits, 10);
AreEqual(3, hits.TotalHits);
AreEqual("2", r.Document(hits.ScoreDocs[0].Doc).Get("id"));
AreEqual("1", r.Document(hits.ScoreDocs[1].Doc).Get("id"));
AreEqual("3", r.Document(hits.ScoreDocs[2].Doc).Get("id"));
string expl = rescorer.Explain(searcher, searcher.Explain(query, hits.ScoreDocs[0].Doc), hits.ScoreDocs[0].Doc).ToString();
// Confirm the explanation breaks out the individual
// variables:
IsTrue(expl.Contains("= variable \"popularity\""));
// Confirm the explanation includes first pass details:
IsTrue(expl.Contains("= first pass score"));
IsTrue(expl.Contains("body:contents in"));
}
internal MultiMostSimilarEstimator(long[] toItemIDs, ItemSimilarity similarity, Rescorer <Tuple <long, long> > rescorer, bool excludeItemIfNotSimilarToAll)
{
this.toItemIDs = toItemIDs;
this.similarity = similarity;
this.rescorer = rescorer;
this.excludeItemIfNotSimilarToAll = excludeItemIfNotSimilarToAll;
}
internal MostSimilarEstimator(User toUser,
UserCorrelation correlation,
Rescorer <Pair <User, User> > rescorer)
{
this.toUser = toUser;
this.correlation = correlation;
this.rescorer = rescorer;
}
private List <RecommendedItem> mostSimilarItems(long itemID,
IEnumerator <long> possibleItemIDs,
int howMany,
Rescorer <Tuple <long, long> > rescorer)
{
TopItems.Estimator <long> estimator = new MostSimilarEstimator(itemID, getSimilarity(), rescorer);
return(TopItems.getTopItems(howMany, possibleItemIDs, null, estimator));
}
internal MostSimilarEstimator(Item toItem,
ItemCorrelation correlation,
Rescorer <Pair <Item, Item> > rescorer)
{
this.toItem = toItem;
this.correlation = correlation;
this.rescorer = rescorer;
}
public MultiMostSimilarEstimator(List <Item> toItems,
ItemCorrelation correlation,
Rescorer <Pair <Item, Item> > rescorer)
{
this.toItems = toItems;
this.correlation = correlation;
this.rescorer = rescorer;
}
public ByRescoreComparator(Rescorer <Item> rescorer)
{
if (rescorer == null)
{
throw new ArgumentNullException("rescorer is null");
}
this.rescorer = rescorer;
}
public void TestItemRescorer()
{
Rescorer <Item> rescorer = NullRescorer <Item> .Instance;
Assert.IsNotNull(rescorer);
Item item = new GenericItem <String>("test");
Assert.AreEqual(1.0, rescorer.Rescore(item, 1.0));
Assert.AreEqual(1.0, rescorer.Rescore(null, 1.0));
Assert.AreEqual(0.0, rescorer.Rescore(item, 0.0));
Assert.IsTrue(Double.IsNaN(rescorer.Rescore(item, Double.NaN)));
}
public void TestUserRescorer()
{
Rescorer <User> rescorer = NullRescorer <User> .Instance;
Assert.IsNotNull(rescorer);
User user = new GenericUser <String>("test", new List <Preference>());
Assert.AreEqual(1.0, rescorer.Rescore(user, 1.0));
Assert.AreEqual(1.0, rescorer.Rescore(null, 1.0));
Assert.AreEqual(0.0, rescorer.Rescore(user, 0.0));
Assert.IsTrue(Double.IsNaN(rescorer.Rescore(user, Double.NaN)));
}
public IList <RecommendedItem> MostSimilarItems(Object itemID,
int howMany,
Rescorer <Pair <Item, Item> > rescorer)
{
if (rescorer == null)
{
throw new ArgumentNullException("rescorer is null");
}
Item toItem = this.DataModel.GetItem(itemID);
TopItems.Estimator <Item> estimator = new MostSimilarEstimator(toItem, correlation, rescorer);
return(DoMostSimilarItems(itemID, howMany, estimator));
}
public IList <User> MostSimilarUsers(Object userID,
int howMany,
Rescorer <Pair <User, User> > rescorer)
{
if (rescorer == null)
{
throw new ArgumentNullException("rescorer is null");
}
User toUser = this.DataModel.GetUser(userID);
TopItems.Estimator <User> estimator = new MostSimilarEstimator(toUser, correlation, rescorer);
return(DoMostSimilarUsers(userID, howMany, estimator));
}
public void TestRecommender()
{
AtomicInteger recommendCount = new AtomicInteger();
Recommender mockRecommender = new MockRecommender(recommendCount);
Recommender cachingRecommender = new CachingRecommender(mockRecommender);
cachingRecommender.Recommend("1", 1);
Assert.AreEqual(1, recommendCount.Value);
cachingRecommender.Recommend("2", 1);
Assert.AreEqual(2, recommendCount.Value);
cachingRecommender.Recommend("1", 1);
Assert.AreEqual(2, recommendCount.Value);
cachingRecommender.Recommend("2", 1);
Assert.AreEqual(2, recommendCount.Value);
cachingRecommender.Refresh();
cachingRecommender.Recommend("1", 1);
Assert.AreEqual(3, recommendCount.Value);
cachingRecommender.Recommend("2", 1);
Assert.AreEqual(4, recommendCount.Value);
cachingRecommender.Recommend("3", 1);
Assert.AreEqual(5, recommendCount.Value);
// Results from this recommend() method can't be cached:
Rescorer <Item> rescorer = NullRescorer <Item> .Instance;
cachingRecommender.Refresh();
cachingRecommender.Recommend("1", 1, rescorer);
Assert.AreEqual(6, recommendCount.Value);
cachingRecommender.Recommend("2", 1, rescorer);
Assert.AreEqual(7, recommendCount.Value);
cachingRecommender.Recommend("1", 1, rescorer);
Assert.AreEqual(8, recommendCount.Value);
cachingRecommender.Recommend("2", 1, rescorer);
Assert.AreEqual(9, recommendCount.Value);
cachingRecommender.Refresh();
cachingRecommender.EstimatePreference("test1", "1");
Assert.AreEqual(10, recommendCount.Value);
cachingRecommender.EstimatePreference("test1", "2");
Assert.AreEqual(11, recommendCount.Value);
cachingRecommender.EstimatePreference("test1", "2");
Assert.AreEqual(11, recommendCount.Value);
}
public IList <RecommendedItem> Recommend(Object userID, int howMany, Rescorer <Item> rescorer)
{
// Hmm, hard to recommendationCache this since the rescorer may change
return(recommender.Recommend(userID, howMany, rescorer));
}
public IList <RecommendedItem> Recommend(Object userID,
int howMany,
Rescorer <Item> rescorer)
{
return(Recommend(userID, howMany));
}
public virtual IList <RecommendedItem> Recommend(Object userID, int howMany, Rescorer <Item> rescorer)
{
throw new NotImplementedException("The method or operation is not implemented.");
}
public virtual long[] mostSimilarUserIDs(long userID, int howMany, Rescorer <Tuple <long, long> > rescorer)
{
TopItems.Estimator <long> estimator = new MostSimilarEstimator(userID, this.similarity, rescorer);
return(this.doMostSimilarUsers(howMany, estimator));
}
public List <RecommendedItem> mostSimilarItems(long[] itemIDs, int howMany, Rescorer <Tuple <long, long> > rescorer)
{
TopItems.Estimator <long> estimator = new MultiMostSimilarEstimator(itemIDs, this.similarity, rescorer, EXCLUDE_ITEM_IF_NOT_SIMILAR_TO_ALL_BY_DEFAULT);
return(this.doMostSimilarItems(itemIDs, howMany, estimator));
}
internal MostSimilarEstimator(long toUserID, UserSimilarity similarity, Rescorer <Tuple <long, long> > rescorer)
{
this.toUserID = toUserID;
this.similarity = similarity;
this.rescorer = rescorer;
}
public override IList <RecommendedItem> Recommend(Object userID, int howMany, Rescorer <Item> rescorer)
{
if (userID == null)
{
throw new ArgumentNullException("userID is null");
}
if (howMany < 1)
{
throw new ArgumentException("howMany must be at least 1");
}
if (rescorer == null)
{
throw new ArgumentNullException("rescorer is null");
}
if (log.IsDebugEnabled)
{
log.DebugFormat("Recommending items for user ID '{0}'", userID);
}
User theUser = this.DataModel.GetUser(userID);
ICollection <User> theNeighborhood = neighborhood.GetUserNeighborhood(userID);
if (log.IsDebugEnabled)
{
log.DebugFormat("UserNeighborhood is: {0} ", neighborhood);
}
if (theNeighborhood.Count == 0)
{
return(new List <RecommendedItem>());
}
ISet <Item> allItems = GetAllOtherItems(theNeighborhood, theUser);
if (log.IsDebugEnabled)
{
log.Debug("Items in Neighborhood which user doesn't prefer already are: " + allItems);
}
TopItems.Estimator <Item> estimator = new Estimator(this, theUser, theNeighborhood);
IList <RecommendedItem> topItems = TopItems.GetTopItems(howMany, allItems, rescorer, estimator);
if (log.IsDebugEnabled)
{
log.Debug("Recommendations are: " + topItems); // TODO: Format this better
}
return(topItems);
}
public MostSimilarEstimator(long toItemID, ItemSimilarity similarity, Rescorer <Tuple <long, long> > rescorer)
{
this.toItemID = toItemID;
this.similarity = similarity;
this.rescorer = rescorer;
}
public List <RecommendedItem> mostSimilarItems(long[] itemIDs, int howMany, Rescorer <Tuple <long, long> > rescorer, bool excludeItemIfNotSimilarToAll)
{
TopItems.Estimator <long> estimator = new MultiMostSimilarEstimator(itemIDs, this.similarity, rescorer, excludeItemIfNotSimilarToAll);
return(this.doMostSimilarItems(itemIDs, howMany, estimator));
}
public List <RecommendedItem> mostSimilarItems(long itemID, int howMany, Rescorer <Tuple <long, long> > rescorer)
{
TopItems.Estimator <long> estimator = new MostSimilarEstimator(itemID, this.similarity, rescorer);
return(this.doMostSimilarItems(new long[] { itemID }, howMany, estimator));
}
/**
* {@inheritDoc}
*/
public override IList <RecommendedItem> Recommend(Object userID, int howMany, Rescorer <Item> rescorer)
{
if (userID == null)
{
throw new ArgumentNullException("userID is null");
}
if (howMany < 1)
{
throw new ArgumentException("howMany must be at least 1");
}
if (rescorer == null)
{
throw new ArgumentNullException("rescorer is null");
}
if (log.IsDebugEnabled)
{
log.Debug("Recommending items for user ID '" + userID + '\'');
}
User theUser = this.DataModel.GetUser(userID);
ISet <Item> allItems = diffStorage.GetRecommendableItems(userID);
TopItems.Estimator <Item> estimator = new Estimator(this, theUser);
IList <RecommendedItem> topItems = TopItems.GetTopItems(howMany, allItems, rescorer, estimator);
if (log.IsDebugEnabled)
{
log.Debug("Recommendations are: " + topItems);
}
return(topItems);
}
public static List <User> GetTopUsers(int howMany,
IEnumerable <User> allUsers,
Rescorer <User> rescorer,
Estimator <User> estimator)
{
LinkedList <SimilarUser> topUsers = new LinkedList <SimilarUser>();
bool full = false;
foreach (User user in allUsers)
{
if (rescorer.IsFiltered(user))
{
continue;
}
double similarity = estimator.Estimate(user);
double rescoredSimilarity = rescorer.Rescore(user, similarity);
LinkedListNode <SimilarUser> node = topUsers.Last;
if (!double.IsNaN(rescoredSimilarity) &&
(!full || rescoredSimilarity > node.Value.Similarity))
{
//SimilarUser _user = new SimilarUser(user, similarity);
SimilarUser _user = new SimilarUser(user, rescoredSimilarity);
if (node == null)
{
topUsers.AddLast(_user);
}
else if (node.Previous == null) // 1 node
{
if (rescoredSimilarity > node.Value.Similarity)
{
topUsers.AddAfter(node, _user);
}
else
{
topUsers.AddBefore(node, _user);
}
}
else
{
while (node != null && node.Previous != null && (node != topUsers.First))
{
node = node.Previous;
if (rescoredSimilarity <= node.Value.Similarity)
{
topUsers.AddBefore(node, _user);
break;
}
}
}
if (full)
{
topUsers.RemoveLast();
}
else if (topUsers.Count > howMany)
{
full = true;
topUsers.RemoveLast();
}
}
}
List <User> result = new List <User>(topUsers.Count);
foreach (SimilarUser similarUser in topUsers)
{
result.Add(similarUser.User);
}
return(result);
}
public virtual IList<RecommendedItem> Recommend(Object userID, int howMany, Rescorer<Item> rescorer)
{
throw new NotImplementedException("The method or operation is not implemented.");
}
public static IList <RecommendedItem> GetTopItems(int howMany,
IEnumerable <Item> allItems,
Rescorer <Item> rescorer,
Estimator <Item> estimator)
{
if (allItems == null || rescorer == null || estimator == null)
{
throw new ArgumentNullException("argument is null");
}
LinkedList <RecommendedItem> topItems = new LinkedList <RecommendedItem>();
bool full = false;
foreach (Item item in allItems)
{
if (item.IsRecommendable && !rescorer.IsFiltered(item))
{
double preference = estimator.Estimate(item);
double rescoredPref = rescorer.Rescore(item, preference);
LinkedListNode <RecommendedItem> node = topItems.Last;
if (!Double.IsNaN(rescoredPref) &&
(!full || rescoredPref > node.Value.Value))
{
// I think this is faster than Collections.binarySearch() over a LinkedList since our
// comparisons are cheap, which binarySearch() economizes at the expense of more traversals.
// We also know that the right position tends to be at the end of the list.
while (node != null && node.Previous != null)
{
node = node.Previous;
if (rescoredPref <= node.Value.Value)
{
node = node.Next;
break;
}
if (node == topItems.First)
{
break;
}
}
RecommendedItem newItem = new GenericRecommendedItem(item, rescoredPref);
if (node == null)
{
topItems.AddFirst(newItem);
}
else if (topItems.Count == 1)
{
// special handling in this case is to avoid problems
// with negative preferences. Imagine -0.3 being added
// first followed by -0.6. If we simply did AddAfter,
// those items would be out of sequence - cc
if (rescoredPref > node.Value.Value)
{
topItems.AddAfter(node, newItem);
}
else
{
topItems.AddBefore(node, newItem);
}
}
else
{
topItems.AddAfter(node, newItem);
}
if (full)
{
topItems.RemoveLast();
}
else if (topItems.Count > howMany)
{
full = true;
topItems.RemoveLast();
}
}
}
}
List <RecommendedItem> result = new List <RecommendedItem>(topItems.Count);
foreach (RecommendedItem item in topItems)
{
result.Add(item);
}
return(result);
}
public IList<RecommendedItem> Recommend( Object userID,
int howMany,
Rescorer<Item> rescorer)
{
return Recommend(userID, howMany);
}
/**
* {@inheritDoc}
*/
public override IList <RecommendedItem> Recommend(Object userID, int howMany, Rescorer <Item> rescorer)
{
if (userID == null || rescorer == null)
{
throw new ArgumentNullException("userID or rescorer is null");
}
if (howMany < 1)
{
throw new ArgumentException("howMany must be at least 1");
}
CheckClustersBuilt();
if (log.IsDebugEnabled)
{
log.Debug("Recommending items for user ID '" + userID + '\'');
}
IList <RecommendedItem> recommended;
if (!topRecsByUserID.TryGetValue(userID, out recommended))
{
return(new List <RecommendedItem>());
}
User theUser = this.DataModel.GetUser(userID);
List <RecommendedItem> rescored = new List <RecommendedItem>(recommended.Count);
// Only add items the user doesn't already have a preference for.
// And that the rescorer doesn't "reject".
foreach (RecommendedItem recommendedItem in recommended)
{
Item item = recommendedItem.Item;
if (rescorer.IsFiltered(item))
{
continue;
}
if (theUser.GetPreferenceFor(item.ID) == null &&
!double.IsNaN(rescorer.Rescore(item, recommendedItem.Value)))
{
rescored.Add(recommendedItem);
}
}
rescored.Sort(new ByRescoreComparator(rescorer));
return(rescored);
}