private static async Task DownloadLeaders2()
{
//Thread.CurrentThread.Name = "leaders";
log.Info("Initializing download leaders info...");
RateLimit.RateLimitTrackerOption = RateLimitTrackerOptions.TrackAndAwait;
Auth.SetUserCredentials("W628b4dGSYkBy3oxaBou0Dp7W", "4DSRpc2uPKkw2yTn0Q5AdaO3zkHNJK7p2mA2f8M05alTQl8ulX", "1020526939-g4betm5uhETG7Eh9b4JipGjyUsaQtSvTJDZuDyS", "QIjKTi5NkqAbSwJwwtc9HcpzTnNpGrRxJKcns0CbTMXJA");
var user = User.GetLoggedUser();
log.Info("Logged as " + user.ScreenName);
IMongoClient mclient = new MongoClient();
IMongoDatabase mdb = mclient.GetDatabase("twitter");
var leaders = mdb.GetCollection<BsonDocument>("users");
var proc = mdb.GetCollection<BsonDocument>("usersproc");
var filter = new BsonDocument();
while (true)
{
try
{
using (var cursor = await leaders.FindAsync(filter))
{
while (await cursor.MoveNextAsync())
{
var batch = cursor.Current;
foreach (var leader in batch)
{
try
{
long id = long.Parse(leader["id_str"].AsString);
var filterproc = Builders<BsonDocument>.Filter.Eq("id", id);
var res = await proc.Find(filterproc).ToListAsync();
//log.Info("procesing leader {0}", id);
Console.WriteLine(string.Format("procesing leader {0}", id));
if (res.Count > 0)
continue;
string u = leader["screen_name"].AsString;
//TwitterProfileHandler profileHandler = new TwitterProfileHandlerMongo();
//profileHandler.Download(u);
TwitterHomelineHandlerMongo homeHandler = new TwitterHomelineHandlerMongo();
homeHandler.MaxTweets = 200;
homeHandler.Download(u);
//TwitterRetweetHandler retweeterHandler = new TwitterRetweetHandlerMongo();
//retweeterHandler.Download(u);
BsonDocument d = new BsonDocument();
d["id"] = id;
await proc.InsertOneAsync(d);
}
catch (Exception ex)
{
log.Error(ex.Message, ex);
}
}
}
}
Thread.Sleep(5000);
}
catch (Exception exx)
{
log.Error(exx.Message);
}
}
}
private static async Task DownloadLeaders()
{
//Thread.CurrentThread.Name = "leaders";
log.Info("Initializing download leaders info...");
RateLimit.RateLimitTrackerOption = RateLimitTrackerOptions.TrackAndAwait;
Auth.SetUserCredentials("W628b4dGSYkBy3oxaBou0Dp7W", "4DSRpc2uPKkw2yTn0Q5AdaO3zkHNJK7p2mA2f8M05alTQl8ulX", "1020526939-g4betm5uhETG7Eh9b4JipGjyUsaQtSvTJDZuDyS", "QIjKTi5NkqAbSwJwwtc9HcpzTnNpGrRxJKcns0CbTMXJA");
var user = User.GetLoggedUser();
log.Info("Logged as " + user.ScreenName);
IMongoClient mclient = new MongoClient();
IMongoDatabase mdb = mclient.GetDatabase("twitter");
var leaders = mdb.GetCollection<BsonDocument>("leaders");
var filter = new BsonDocument();
while (key != ConsoleKey.Escape)
{
using (var cursor = await leaders.FindAsync(filter))
{
while (await cursor.MoveNextAsync())
{
var batch = cursor.Current;
foreach (var leader in batch)
{
try
{
string u = leader["user"].AsString;
log.Trace("procesing leader {0}", u);
TwitterProfileHandler profileHandler = new TwitterProfileHandlerMongo();
profileHandler.Download(u);
TwitterHomelineHandlerMongo homeHandler = new TwitterHomelineHandlerMongo();
homeHandler.MaxTweets = 200;
homeHandler.Download(u);
//TwitterRetweetHandler retweeterHandler = new TwitterRetweetHandlerMongo();
//retweeterHandler.Download(u);
}catch(Exception ex)
{
log.Error(ex.Message, ex);
}
}
}
}
Thread.Sleep(5000);
}
}