public IList <string> MineTweets(ITwitterAuthentificationResponse twitterAuthentificationResponse, ITwitterMinerQuery minerQuery, long numberOfTweets, long?sinceId = null)
{
long max_id = -1;
long numberOfTweetsProcessed = 0;
List <string> minedTwits = new List <string>();
while (numberOfTweetsProcessed < numberOfTweets)
{
string twitterResult;
if (max_id <= 0)
{
if (!sinceId.HasValue)
{
twitterResult = SearchByQuery(twitterAuthentificationResponse, minerQuery.Build());
}
else
{
var query = minerQuery.WithSinceId(sinceId.Value).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
}
else
{
if (!sinceId.HasValue)
{
var query = minerQuery.WithMaxId(max_id - 1).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
else
{
var query = minerQuery.WithMaxId(max_id - 1).WithSinceId(sinceId.Value).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
}
dynamic deserializeObject = JsonConvert.DeserializeObject(twitterResult);
if (deserializeObject.statuses.Count == 0)
{
return(minedTwits);
}
numberOfTweetsProcessed += deserializeObject.search_metadata.count.Value;
max_id = deserializeObject.statuses.Last().id;
minedTwits.Add(twitterResult);
}
return(minedTwits);
}
private void ProcessData(ITwitterAuthentificationResponse twitterAuthentificationResponse, ITwitterMinerQuery minerQuery,
long numberOfTweets, Action <string> jsonProcessor, long?sinceId, bool runAsync)
{
Stopwatch stopWatch = new Stopwatch();
long max_id = -1;
long numberOfTweetsProcessed = 0;
int numberOfCalls = 0;
stopWatch.Start();
while (numberOfTweetsProcessed < numberOfTweets)
{
string twitterResult;
if (max_id <= 0)
{
if (!sinceId.HasValue)
{
twitterResult = SearchByQuery(twitterAuthentificationResponse, minerQuery.Build());
}
else
{
var query = minerQuery.WithSinceId(sinceId.Value).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
}
else
{
if (!sinceId.HasValue)
{
var query = minerQuery.WithMaxId(max_id - 1).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
else
{
var query = minerQuery.WithMaxId(max_id - 1).WithSinceId(sinceId.Value).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
}
numberOfCalls++;
dynamic deserializeObject = JsonConvert.DeserializeObject(twitterResult);
if (deserializeObject.statuses.Count == 0)
{
break;
}
numberOfTweetsProcessed += deserializeObject.search_metadata.count.Value;
max_id = deserializeObject.statuses.Last.id.Value;
if (runAsync)
{
Task.Factory.StartNew(() => jsonProcessor(twitterResult));
}
else
{
jsonProcessor(twitterResult);
}
if (numberOfCalls >= 450)
{
stopWatch.Stop();
long elapsedMilliseconds = 15L * 1000L * 60L - stopWatch.ElapsedMilliseconds;
Console.WriteLine("going to sleep...");
Thread.Sleep((int)(elapsedMilliseconds >= 0L?elapsedMilliseconds:0L));
numberOfCalls = 0;
stopWatch.Restart();
}
}
}