/// <summary> /// Craetes/updates an index /// </summary> /// <param name="entity"></param> /// <returns></returns> public RetreaveIndex SaveOrUpdate(RetreaveIndex entity) { if (_database.IsNew(entity)) { _database.Insert(entity); } else { _database.Update(entity); } //update the associated users _database.Execute("Delete from Users_Indexes where indexId=@0", entity.IndexId); foreach (RegisteredUser user in entity.AssociatedUsers) { if (_database.IsNew(user)) { throw new Exception("You must save the User entity first"); } _database.Execute("Insert into Users_indexes (UserId, IndexId) values (@0, @1)", user.UserId, entity.IndexId); } return(entity); }
/// <summary> /// Perform the indexing on a twitter stream /// </summary> /// <param name="index">the stream to index</param> public void Process(RetreaveIndex index) { //get the users tweets TwitterStreamRetriever retreaver = new TwitterStreamRetriever(index.AssociatedUsers.First().AuthDetails); IList <Tweet> tweets = retreaver.GetAllTweets(); IList <Tweet> tweetsWithUrls = new List <Tweet>(); //find the ones with URLs foreach (Tweet tweet in tweets) { if (tweet.ContainsUrl()) { tweetsWithUrls.Add(tweet); } } Console.WriteLine("Found " + tweetsWithUrls.Count + " tweets out of " + tweets.Count + " Have URLs"); //foreach one, retrieve the HTML); foreach (Tweet tweetWithUrl in tweetsWithUrls) { //index the tweet in the tweet index _tweetIndexer.IndexTweet(tweetWithUrl, index.IndexStreamIdentifier); //index all the URl's from tweet - this should be done later though. // _urlIndexer.IndexUrlsInTweet(tweetWithUrl, index); } ServiceLayer.IndexQueuerService.MarkIndexComplete(index.IndexId); }
/// <summary> /// Fills out the rest of the properties on an index object /// by executing more queries /// </summary> /// <param name="returnedIndex">an index returned in its basic form</param> /// <returns></returns> public RetreaveIndex FillResult(RetreaveIndex returnedIndex) { if (returnedIndex == null) { return(null); } returnedIndex.AssociatedUsers = _registeredUserDao.GetUsersByIndex(returnedIndex).ToList(); return(returnedIndex); }
/// <summary> /// Gets the user by their index /// </summary> /// <param name="index"></param> /// <returns></returns> internal IEnumerable <RegisteredUser> GetUsersByIndex(RetreaveIndex index) { var sql = Sql.Builder .Append(" Select U.*, AUTH.*") .Append("From [RetrEave].[dbo].AuthenticationDetails AUTH ") .Append("JOIN [RetrEave].[dbo].[Users] U on AUTH.AuthenticationDetailsId = U.AuthenticationDetails") .Append("JOIN [RetrEave].[dbo].Users_Indexes") .Append("ON U.UserId = Users_Indexes.UserId ") .Where("Users_Indexes.IndexId = @0", index.IndexId); return(GetUsersByQuery(sql)); }
/// <summary> /// Adds a users personal feed to the queue for indexing /// </summary> /// <param name="user">the user to queue</param> public void QueueUserStreamIndex(RegisteredUser user) { //add a a new StreamIndex to the users indexes RetreaveIndex streamIndex = new RetreaveIndex() { IndexType = IndexType.TwitterStreamIndex, Name = "Twitter Stream for " + user.UserName, Active = true, DateAdded = DateTime.Now, IndexStreamIdentifier = user.TwitterId.ToString() }; streamIndex.AssociatedUsers.Add(user); _indexDao.SaveOrUpdate(streamIndex); }
static void Main(string[] args) { //initialize the indexes in case they haven't been made yet InitializeIndexes(); //indexer thread Thread indexerThread = new Thread(() => { while (_running) { RecentTweetsProcessor processor = new RecentTweetsProcessor(); StreamQueuedTweetsProcessor streamProcessor = new StreamQueuedTweetsProcessor(); //process the new indexes RetreaveIndex newIndex = ServiceLayer.IndexQueuerService.GetNextIndexToProcess(); if (newIndex != null) { //do the initial indexing of the previous tweets for convenience. if (newIndex.IndexType == IndexType.TwitterStreamIndex) { Console.WriteLine("Processing new index name " + newIndex.Name); processor.Process(newIndex); } } // read through the tweets queued by the streamer streamProcessor.ProcessNextInQueue(); Thread.Sleep(1 * 1000); } }); Thread streamerThread = new Thread(() => { StreamingTwitterFeedRetriever streamer = new StreamingTwitterFeedRetriever(); //get users tweets to stream List <RetreaveIndex> streamingIndexes = new List <RetreaveIndex>(); while (_running) { //check for new feeds to stream List <RetreaveIndex> indexesFromDatabase = ServiceLayer.IndexQueuerService.GetUserIndexesToStream().ToList(); if (indexesFromDatabase.Count > streamingIndexes.Count) { streamingIndexes = indexesFromDatabase; //if found one, start a new stream and stop old one streamer.Running = false; streamer.StreamUsersTweets(streamingIndexes); } } streamer.Running = false; }); indexerThread.Start(); streamerThread.Start(); Console.WriteLine("Indexer Running.. Press any key to exit"); Console.ReadLine(); _running = false; }
/// <summary> /// Deletes the index /// </summary> /// <param name="entity"></param> public void Delete(RetreaveIndex entity) { _database.Delete(entity); }