示例#1
0
        /// <summary>
        /// Perform the indexing on a twitter stream
        /// </summary>
        /// <param name="index">the stream to index</param>
        public void Process(RetreaveIndex index)
        {
            //get the users tweets
            TwitterStreamRetriever retreaver = new TwitterStreamRetriever(index.AssociatedUsers.First().AuthDetails);
            IList<Tweet> tweets = retreaver.GetAllTweets();

            IList<Tweet> tweetsWithUrls = new List<Tweet>();
            //find the ones with URLs
            foreach (Tweet tweet in tweets)
            {
                if (tweet.ContainsUrl())
                    tweetsWithUrls.Add(tweet);
            }

            Console.WriteLine("Found " + tweetsWithUrls.Count + " tweets out of " + tweets.Count + " Have URLs");

            //foreach one, retrieve the HTML);
            foreach (Tweet tweetWithUrl in tweetsWithUrls)
            {

                //index the tweet in the tweet index
                _tweetIndexer.IndexTweet(tweetWithUrl, index.IndexStreamIdentifier);

                //index all the URl's from tweet - this should be done later though.
               // _urlIndexer.IndexUrlsInTweet(tweetWithUrl, index);
            }

            ServiceLayer.IndexQueuerService.MarkIndexComplete(index.IndexId);
        }
示例#2
0
        /// <summary>
        /// Gets the user by their index
        /// </summary>
        /// <param name="index"></param>
        /// <returns></returns>
        internal IEnumerable<RegisteredUser> GetUsersByIndex(RetreaveIndex index)
        {
            var sql =  Sql.Builder
                .Append(" Select U.*, AUTH.*")
                .Append("From [RetrEave].[dbo].AuthenticationDetails AUTH ")
                .Append("JOIN [RetrEave].[dbo].[Users] U on AUTH.AuthenticationDetailsId = U.AuthenticationDetails")
                .Append("JOIN [RetrEave].[dbo].Users_Indexes")
                .Append("ON U.UserId = Users_Indexes.UserId ")
                .Where("Users_Indexes.IndexId = @0", index.IndexId);

            return GetUsersByQuery(sql);
        }
示例#3
0
        /// <summary>
        /// Adds a users personal feed to the queue for indexing
        /// </summary>
        /// <param name="user">the user to queue</param>
        public void QueueUserStreamIndex(RegisteredUser user)
        {
            //add a a new StreamIndex to the users indexes
            RetreaveIndex streamIndex = new RetreaveIndex()
                              {
                                  IndexType = IndexType.TwitterStreamIndex,
                                  Name = "Twitter Stream for " + user.UserName,
                                  Active = true,
                                  DateAdded = DateTime.Now,
                                  IndexStreamIdentifier = user.TwitterId.ToString()
                              };
            streamIndex.AssociatedUsers.Add(user);

            _indexDao.SaveOrUpdate(streamIndex);
        }