Esempio n. 1
0
        private static async Task DownloadLeaders()
        {
            //Thread.CurrentThread.Name = "leaders";
            log.Info("Initializing download leaders info...");
            RateLimit.RateLimitTrackerOption = RateLimitTrackerOptions.TrackAndAwait;
            Auth.SetUserCredentials("W628b4dGSYkBy3oxaBou0Dp7W", "4DSRpc2uPKkw2yTn0Q5AdaO3zkHNJK7p2mA2f8M05alTQl8ulX", "1020526939-g4betm5uhETG7Eh9b4JipGjyUsaQtSvTJDZuDyS", "QIjKTi5NkqAbSwJwwtc9HcpzTnNpGrRxJKcns0CbTMXJA");
            var user = User.GetLoggedUser();
            log.Info("Logged as " + user.ScreenName);
            

            IMongoClient mclient = new MongoClient();
            IMongoDatabase mdb = mclient.GetDatabase("twitter");
            var leaders = mdb.GetCollection<BsonDocument>("leaders");
            var filter = new BsonDocument();

            while (key != ConsoleKey.Escape)
            {
                using (var cursor = await leaders.FindAsync(filter))
                {
                    while (await cursor.MoveNextAsync())
                    {
                        var batch = cursor.Current;
                        foreach (var leader in batch)
                        {

                            try
                            {
                                string u = leader["user"].AsString;
                                log.Trace("procesing leader {0}", u);
                                TwitterProfileHandler profileHandler = new TwitterProfileHandlerMongo();
                                profileHandler.Download(u);
                                TwitterHomelineHandlerMongo homeHandler = new TwitterHomelineHandlerMongo();
                                homeHandler.MaxTweets = 200;
                                homeHandler.Download(u);
                                //TwitterRetweetHandler retweeterHandler = new TwitterRetweetHandlerMongo();
                                //retweeterHandler.Download(u);
                            }catch(Exception ex)
                            {
                                log.Error(ex.Message, ex);
                            }
                        }
                    }
                }

                Thread.Sleep(5000);
            }
        }
Esempio n. 2
0
        private static async Task Dr(IMongoCollection<BsonDocument> leaders, BsonDocument filter, IMongoCollection<BsonDocument> users, IMongoCollection<BsonDocument> leadersbak)
        {
            using (var cursor = await leaders.FindAsync(filter))
            {

                while (await cursor.MoveNextAsync())
                {
                    var batch = cursor.Current;
                    foreach (var rt in batch)
                    {
                        BsonArray bRetweeters = rt["retweeters_id"].AsBsonArray;

                        //if (bRetweeters.Count == 0)
                        //    continue;

                        foreach (var bUser in bRetweeters)
                        {
                            var lUser = bUser.ToInt64();
                            try
                            {

                                log.Info("readling retweeter {0}", lUser);
                                var usersFilter = Builders<BsonDocument>.Filter.Eq("id", lUser);
                                var result = await users.Find(usersFilter).ToListAsync();
                                if (result.Count > 0)
                                    continue;

                                log.Info("downloading retweeter {0}", lUser);
                                TwitterProfileHandler profileHandler = new TwitterProfileHandlerMongo();
                                string screenName = profileHandler.Download(lUser);
                                //TwitterHomelineHandler homeHandler = new TwitterHomelineHandlerMongo();
                                //homeHandler.Download(screenName);
                            }
                            catch (Exception ex)
                            {
                                string msg = "Error trying to download profile for user:{0}";
                                msg = string.Format(msg, lUser);
                                log.Error(ex, msg);
                            }
                        }

                        //rt["done"] = true;
                        //var filterupdate = Builders<BsonDocument>.Filter.Eq("_id", rt["_id"]);
                        //var update = Builders<BsonDocument>.Update.Push("done", true);
                        //await leaders.UpdateOneAsync(filterupdate, rt);
                        await leadersbak.InsertOneAsync(rt);
                        var filterbak = Builders<BsonDocument>.Filter.Eq("_id", rt["_id"]);
                        await leaders.DeleteOneAsync(filterbak);
                    }
                }
            }

            Thread.Sleep(5000);
        }