/// <summary>
/// Craetes/updates an index
/// </summary>
/// <param name="entity"></param>
/// <returns></returns>
public RetreaveIndex SaveOrUpdate(RetreaveIndex entity)
{
if (_database.IsNew(entity))
{
_database.Insert(entity);
}
else
{
_database.Update(entity);
}
//update the associated users
_database.Execute("Delete from Users_Indexes where indexId=@0", entity.IndexId);
foreach (RegisteredUser user in entity.AssociatedUsers)
{
if (_database.IsNew(user))
{
throw new Exception("You must save the User entity first");
}
_database.Execute("Insert into Users_indexes (UserId, IndexId) values (@0, @1)", user.UserId,
entity.IndexId);
}
return(entity);
}
/// <summary>
/// Perform the indexing on a twitter stream
/// </summary>
/// <param name="index">the stream to index</param>
public void Process(RetreaveIndex index)
{
//get the users tweets
TwitterStreamRetriever retreaver = new TwitterStreamRetriever(index.AssociatedUsers.First().AuthDetails);
IList <Tweet> tweets = retreaver.GetAllTweets();
IList <Tweet> tweetsWithUrls = new List <Tweet>();
//find the ones with URLs
foreach (Tweet tweet in tweets)
{
if (tweet.ContainsUrl())
{
tweetsWithUrls.Add(tweet);
}
}
Console.WriteLine("Found " + tweetsWithUrls.Count + " tweets out of " + tweets.Count + " Have URLs");
//foreach one, retrieve the HTML);
foreach (Tweet tweetWithUrl in tweetsWithUrls)
{
//index the tweet in the tweet index
_tweetIndexer.IndexTweet(tweetWithUrl, index.IndexStreamIdentifier);
//index all the URl's from tweet - this should be done later though.
// _urlIndexer.IndexUrlsInTweet(tweetWithUrl, index);
}
ServiceLayer.IndexQueuerService.MarkIndexComplete(index.IndexId);
}
/// <summary>
/// Fills out the rest of the properties on an index object
/// by executing more queries
/// </summary>
/// <param name="returnedIndex">an index returned in its basic form</param>
/// <returns></returns>
public RetreaveIndex FillResult(RetreaveIndex returnedIndex)
{
if (returnedIndex == null)
{
return(null);
}
returnedIndex.AssociatedUsers = _registeredUserDao.GetUsersByIndex(returnedIndex).ToList();
return(returnedIndex);
}
/// <summary>
/// Gets the user by their index
/// </summary>
/// <param name="index"></param>
/// <returns></returns>
internal IEnumerable <RegisteredUser> GetUsersByIndex(RetreaveIndex index)
{
var sql = Sql.Builder
.Append(" Select U.*, AUTH.*")
.Append("From [RetrEave].[dbo].AuthenticationDetails AUTH ")
.Append("JOIN [RetrEave].[dbo].[Users] U on AUTH.AuthenticationDetailsId = U.AuthenticationDetails")
.Append("JOIN [RetrEave].[dbo].Users_Indexes")
.Append("ON U.UserId = Users_Indexes.UserId ")
.Where("Users_Indexes.IndexId = @0", index.IndexId);
return(GetUsersByQuery(sql));
}
/// <summary>
/// Adds a users personal feed to the queue for indexing
/// </summary>
/// <param name="user">the user to queue</param>
public void QueueUserStreamIndex(RegisteredUser user)
{
//add a a new StreamIndex to the users indexes
RetreaveIndex streamIndex = new RetreaveIndex()
{
IndexType = IndexType.TwitterStreamIndex,
Name = "Twitter Stream for " + user.UserName,
Active = true,
DateAdded = DateTime.Now,
IndexStreamIdentifier = user.TwitterId.ToString()
};
streamIndex.AssociatedUsers.Add(user);
_indexDao.SaveOrUpdate(streamIndex);
}
static void Main(string[] args)
{
//initialize the indexes in case they haven't been made yet
InitializeIndexes();
//indexer thread
Thread indexerThread = new Thread(() =>
{
while (_running)
{
RecentTweetsProcessor processor = new RecentTweetsProcessor();
StreamQueuedTweetsProcessor streamProcessor = new StreamQueuedTweetsProcessor();
//process the new indexes
RetreaveIndex newIndex =
ServiceLayer.IndexQueuerService.GetNextIndexToProcess();
if (newIndex != null)
{
//do the initial indexing of the previous tweets for convenience.
if (newIndex.IndexType == IndexType.TwitterStreamIndex)
{
Console.WriteLine("Processing new index name " +
newIndex.Name);
processor.Process(newIndex);
}
}
// read through the tweets queued by the streamer
streamProcessor.ProcessNextInQueue();
Thread.Sleep(1 * 1000);
}
});
Thread streamerThread = new Thread(() =>
{
StreamingTwitterFeedRetriever streamer = new StreamingTwitterFeedRetriever();
//get users tweets to stream
List <RetreaveIndex> streamingIndexes = new List <RetreaveIndex>();
while (_running)
{
//check for new feeds to stream
List <RetreaveIndex> indexesFromDatabase = ServiceLayer.IndexQueuerService.GetUserIndexesToStream().ToList();
if (indexesFromDatabase.Count > streamingIndexes.Count)
{
streamingIndexes = indexesFromDatabase;
//if found one, start a new stream and stop old one
streamer.Running = false;
streamer.StreamUsersTweets(streamingIndexes);
}
}
streamer.Running = false;
});
indexerThread.Start();
streamerThread.Start();
Console.WriteLine("Indexer Running.. Press any key to exit");
Console.ReadLine();
_running = false;
}
/// <summary>
/// Deletes the index
/// </summary>
/// <param name="entity"></param>
public void Delete(RetreaveIndex entity)
{
_database.Delete(entity);
}
