private static async Task DownloadLeaders() { //Thread.CurrentThread.Name = "leaders"; log.Info("Initializing download leaders info..."); RateLimit.RateLimitTrackerOption = RateLimitTrackerOptions.TrackAndAwait; Auth.SetUserCredentials("W628b4dGSYkBy3oxaBou0Dp7W", "4DSRpc2uPKkw2yTn0Q5AdaO3zkHNJK7p2mA2f8M05alTQl8ulX", "1020526939-g4betm5uhETG7Eh9b4JipGjyUsaQtSvTJDZuDyS", "QIjKTi5NkqAbSwJwwtc9HcpzTnNpGrRxJKcns0CbTMXJA"); var user = User.GetLoggedUser(); log.Info("Logged as " + user.ScreenName); IMongoClient mclient = new MongoClient(); IMongoDatabase mdb = mclient.GetDatabase("twitter"); var leaders = mdb.GetCollection<BsonDocument>("leaders"); var filter = new BsonDocument(); while (key != ConsoleKey.Escape) { using (var cursor = await leaders.FindAsync(filter)) { while (await cursor.MoveNextAsync()) { var batch = cursor.Current; foreach (var leader in batch) { try { string u = leader["user"].AsString; log.Trace("procesing leader {0}", u); TwitterProfileHandler profileHandler = new TwitterProfileHandlerMongo(); profileHandler.Download(u); TwitterHomelineHandlerMongo homeHandler = new TwitterHomelineHandlerMongo(); homeHandler.MaxTweets = 200; homeHandler.Download(u); //TwitterRetweetHandler retweeterHandler = new TwitterRetweetHandlerMongo(); //retweeterHandler.Download(u); }catch(Exception ex) { log.Error(ex.Message, ex); } } } } Thread.Sleep(5000); } }
private static async Task Dr(IMongoCollection<BsonDocument> leaders, BsonDocument filter, IMongoCollection<BsonDocument> users, IMongoCollection<BsonDocument> leadersbak) { using (var cursor = await leaders.FindAsync(filter)) { while (await cursor.MoveNextAsync()) { var batch = cursor.Current; foreach (var rt in batch) { BsonArray bRetweeters = rt["retweeters_id"].AsBsonArray; //if (bRetweeters.Count == 0) // continue; foreach (var bUser in bRetweeters) { var lUser = bUser.ToInt64(); try { log.Info("readling retweeter {0}", lUser); var usersFilter = Builders<BsonDocument>.Filter.Eq("id", lUser); var result = await users.Find(usersFilter).ToListAsync(); if (result.Count > 0) continue; log.Info("downloading retweeter {0}", lUser); TwitterProfileHandler profileHandler = new TwitterProfileHandlerMongo(); string screenName = profileHandler.Download(lUser); //TwitterHomelineHandler homeHandler = new TwitterHomelineHandlerMongo(); //homeHandler.Download(screenName); } catch (Exception ex) { string msg = "Error trying to download profile for user:{0}"; msg = string.Format(msg, lUser); log.Error(ex, msg); } } //rt["done"] = true; //var filterupdate = Builders<BsonDocument>.Filter.Eq("_id", rt["_id"]); //var update = Builders<BsonDocument>.Update.Push("done", true); //await leaders.UpdateOneAsync(filterupdate, rt); await leadersbak.InsertOneAsync(rt); var filterbak = Builders<BsonDocument>.Filter.Eq("_id", rt["_id"]); await leaders.DeleteOneAsync(filterbak); } } } Thread.Sleep(5000); }