示例#1
0
        private static async Task DownloadLeaders2()
        {
            //Thread.CurrentThread.Name = "leaders";
            log.Info("Initializing download leaders info...");
            RateLimit.RateLimitTrackerOption = RateLimitTrackerOptions.TrackAndAwait;
            Auth.SetUserCredentials("W628b4dGSYkBy3oxaBou0Dp7W", "4DSRpc2uPKkw2yTn0Q5AdaO3zkHNJK7p2mA2f8M05alTQl8ulX", "1020526939-g4betm5uhETG7Eh9b4JipGjyUsaQtSvTJDZuDyS", "QIjKTi5NkqAbSwJwwtc9HcpzTnNpGrRxJKcns0CbTMXJA");
            var user = User.GetLoggedUser();
            log.Info("Logged as " + user.ScreenName);


            IMongoClient mclient = new MongoClient();
            IMongoDatabase mdb = mclient.GetDatabase("twitter");
            var leaders = mdb.GetCollection<BsonDocument>("users");
            var proc = mdb.GetCollection<BsonDocument>("usersproc");
            var filter = new BsonDocument();



            while (true)
            {
                try
                {
                    using (var cursor = await leaders.FindAsync(filter))
                    {
                        while (await cursor.MoveNextAsync())
                        {
                            var batch = cursor.Current;
                            foreach (var leader in batch)
                            {

                                try
                                {
                                    long id = long.Parse(leader["id_str"].AsString);
                                    var filterproc = Builders<BsonDocument>.Filter.Eq("id", id);
                                    var res = await proc.Find(filterproc).ToListAsync();
                                    //log.Info("procesing leader {0}", id);
                                    Console.WriteLine(string.Format("procesing leader {0}", id));
                                    if (res.Count > 0)
                                        continue;



                                    string u = leader["screen_name"].AsString;
                                    
                                    //TwitterProfileHandler profileHandler = new TwitterProfileHandlerMongo();
                                    //profileHandler.Download(u);
                                    TwitterHomelineHandlerMongo homeHandler = new TwitterHomelineHandlerMongo();
                                    homeHandler.MaxTweets = 200;
                                    homeHandler.Download(u);
                                    //TwitterRetweetHandler retweeterHandler = new TwitterRetweetHandlerMongo();
                                    //retweeterHandler.Download(u);

                                    BsonDocument d = new BsonDocument();
                                    d["id"] = id;
                                    await proc.InsertOneAsync(d);
                                }
                                catch (Exception ex)
                                {
                                    log.Error(ex.Message, ex);
                                }
                            }
                        }
                    }

                    Thread.Sleep(5000);
                }
                catch (Exception exx)
                {
                    log.Error(exx.Message);
                }
            }
           
        
        }
示例#2
0
        private static async Task DownloadLeaders()
        {
            //Thread.CurrentThread.Name = "leaders";
            log.Info("Initializing download leaders info...");
            RateLimit.RateLimitTrackerOption = RateLimitTrackerOptions.TrackAndAwait;
            Auth.SetUserCredentials("W628b4dGSYkBy3oxaBou0Dp7W", "4DSRpc2uPKkw2yTn0Q5AdaO3zkHNJK7p2mA2f8M05alTQl8ulX", "1020526939-g4betm5uhETG7Eh9b4JipGjyUsaQtSvTJDZuDyS", "QIjKTi5NkqAbSwJwwtc9HcpzTnNpGrRxJKcns0CbTMXJA");
            var user = User.GetLoggedUser();
            log.Info("Logged as " + user.ScreenName);
            

            IMongoClient mclient = new MongoClient();
            IMongoDatabase mdb = mclient.GetDatabase("twitter");
            var leaders = mdb.GetCollection<BsonDocument>("leaders");
            var filter = new BsonDocument();

            while (key != ConsoleKey.Escape)
            {
                using (var cursor = await leaders.FindAsync(filter))
                {
                    while (await cursor.MoveNextAsync())
                    {
                        var batch = cursor.Current;
                        foreach (var leader in batch)
                        {

                            try
                            {
                                string u = leader["user"].AsString;
                                log.Trace("procesing leader {0}", u);
                                TwitterProfileHandler profileHandler = new TwitterProfileHandlerMongo();
                                profileHandler.Download(u);
                                TwitterHomelineHandlerMongo homeHandler = new TwitterHomelineHandlerMongo();
                                homeHandler.MaxTweets = 200;
                                homeHandler.Download(u);
                                //TwitterRetweetHandler retweeterHandler = new TwitterRetweetHandlerMongo();
                                //retweeterHandler.Download(u);
                            }catch(Exception ex)
                            {
                                log.Error(ex.Message, ex);
                            }
                        }
                    }
                }

                Thread.Sleep(5000);
            }
        }