示例#1
0
        public IList <string> MineTweets(ITwitterAuthentificationResponse twitterAuthentificationResponse, ITwitterMinerQuery minerQuery, long numberOfTweets, long?sinceId = null)
        {
            long          max_id = -1;
            long          numberOfTweetsProcessed = 0;
            List <string> minedTwits = new List <string>();

            while (numberOfTweetsProcessed < numberOfTweets)
            {
                string twitterResult;
                if (max_id <= 0)
                {
                    if (!sinceId.HasValue)
                    {
                        twitterResult = SearchByQuery(twitterAuthentificationResponse, minerQuery.Build());
                    }
                    else
                    {
                        var query = minerQuery.WithSinceId(sinceId.Value).Build();
                        twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
                    }
                }
                else
                {
                    if (!sinceId.HasValue)
                    {
                        var query = minerQuery.WithMaxId(max_id - 1).Build();
                        twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
                    }
                    else
                    {
                        var query = minerQuery.WithMaxId(max_id - 1).WithSinceId(sinceId.Value).Build();
                        twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
                    }
                }
                dynamic deserializeObject = JsonConvert.DeserializeObject(twitterResult);
                if (deserializeObject.statuses.Count == 0)
                {
                    return(minedTwits);
                }
                numberOfTweetsProcessed += deserializeObject.search_metadata.count.Value;
                max_id = deserializeObject.statuses.Last().id;
                minedTwits.Add(twitterResult);
            }

            return(minedTwits);
        }
示例#2
0
        private void ProcessData(ITwitterAuthentificationResponse twitterAuthentificationResponse, ITwitterMinerQuery minerQuery,
                                 long numberOfTweets, Action <string> jsonProcessor, long?sinceId, bool runAsync)
        {
            Stopwatch stopWatch = new Stopwatch();
            long      max_id    = -1;
            long      numberOfTweetsProcessed = 0;
            int       numberOfCalls           = 0;

            stopWatch.Start();
            while (numberOfTweetsProcessed < numberOfTweets)
            {
                string twitterResult;
                if (max_id <= 0)
                {
                    if (!sinceId.HasValue)
                    {
                        twitterResult = SearchByQuery(twitterAuthentificationResponse, minerQuery.Build());
                    }
                    else
                    {
                        var query = minerQuery.WithSinceId(sinceId.Value).Build();
                        twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
                    }
                }
                else
                {
                    if (!sinceId.HasValue)
                    {
                        var query = minerQuery.WithMaxId(max_id - 1).Build();
                        twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
                    }
                    else
                    {
                        var query = minerQuery.WithMaxId(max_id - 1).WithSinceId(sinceId.Value).Build();
                        twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
                    }
                }
                numberOfCalls++;

                dynamic deserializeObject = JsonConvert.DeserializeObject(twitterResult);
                if (deserializeObject.statuses.Count == 0)
                {
                    break;
                }
                numberOfTweetsProcessed += deserializeObject.search_metadata.count.Value;
                max_id = deserializeObject.statuses.Last.id.Value;
                if (runAsync)
                {
                    Task.Factory.StartNew(() => jsonProcessor(twitterResult));
                }
                else
                {
                    jsonProcessor(twitterResult);
                }

                if (numberOfCalls >= 450)
                {
                    stopWatch.Stop();
                    long elapsedMilliseconds = 15L * 1000L * 60L - stopWatch.ElapsedMilliseconds;
                    Console.WriteLine("going to sleep...");
                    Thread.Sleep((int)(elapsedMilliseconds >= 0L?elapsedMilliseconds:0L));
                    numberOfCalls = 0;
                    stopWatch.Restart();
                }
            }
        }