/// <summary>
/// Perform the indexing on a twitter stream
/// </summary>
/// <param name="index">the stream to index</param>
public void Process(RetreaveIndex index)
{
//get the users tweets
TwitterStreamRetriever retreaver = new TwitterStreamRetriever(index.AssociatedUsers.First().AuthDetails);
IList<Tweet> tweets = retreaver.GetAllTweets();
IList<Tweet> tweetsWithUrls = new List<Tweet>();
//find the ones with URLs
foreach (Tweet tweet in tweets)
{
if (tweet.ContainsUrl())
tweetsWithUrls.Add(tweet);
}
Console.WriteLine("Found " + tweetsWithUrls.Count + " tweets out of " + tweets.Count + " Have URLs");
//foreach one, retrieve the HTML);
foreach (Tweet tweetWithUrl in tweetsWithUrls)
{
//index the tweet in the tweet index
_tweetIndexer.IndexTweet(tweetWithUrl, index.IndexStreamIdentifier);
//index all the URl's from tweet - this should be done later though.
// _urlIndexer.IndexUrlsInTweet(tweetWithUrl, index);
}
ServiceLayer.IndexQueuerService.MarkIndexComplete(index.IndexId);
}
/// <summary>
/// Gets the user by their index
/// </summary>
/// <param name="index"></param>
/// <returns></returns>
internal IEnumerable<RegisteredUser> GetUsersByIndex(RetreaveIndex index)
{
var sql = Sql.Builder
.Append(" Select U.*, AUTH.*")
.Append("From [RetrEave].[dbo].AuthenticationDetails AUTH ")
.Append("JOIN [RetrEave].[dbo].[Users] U on AUTH.AuthenticationDetailsId = U.AuthenticationDetails")
.Append("JOIN [RetrEave].[dbo].Users_Indexes")
.Append("ON U.UserId = Users_Indexes.UserId ")
.Where("Users_Indexes.IndexId = @0", index.IndexId);
return GetUsersByQuery(sql);
}
/// <summary>
/// Adds a users personal feed to the queue for indexing
/// </summary>
/// <param name="user">the user to queue</param>
public void QueueUserStreamIndex(RegisteredUser user)
{
//add a a new StreamIndex to the users indexes
RetreaveIndex streamIndex = new RetreaveIndex()
{
IndexType = IndexType.TwitterStreamIndex,
Name = "Twitter Stream for " + user.UserName,
Active = true,
DateAdded = DateTime.Now,
IndexStreamIdentifier = user.TwitterId.ToString()
};
streamIndex.AssociatedUsers.Add(user);
_indexDao.SaveOrUpdate(streamIndex);
}