public IList <string> MineTweets(ITwitterAuthentificationResponse twitterAuthentificationResponse, ITwitterMinerQuery minerQuery, long numberOfTweets, long?sinceId = null)
{
long max_id = -1;
long numberOfTweetsProcessed = 0;
List <string> minedTwits = new List <string>();
while (numberOfTweetsProcessed < numberOfTweets)
{
string twitterResult;
if (max_id <= 0)
{
if (!sinceId.HasValue)
{
twitterResult = SearchByQuery(twitterAuthentificationResponse, minerQuery.Build());
}
else
{
var query = minerQuery.WithSinceId(sinceId.Value).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
}
else
{
if (!sinceId.HasValue)
{
var query = minerQuery.WithMaxId(max_id - 1).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
else
{
var query = minerQuery.WithMaxId(max_id - 1).WithSinceId(sinceId.Value).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
}
dynamic deserializeObject = JsonConvert.DeserializeObject(twitterResult);
if (deserializeObject.statuses.Count == 0)
{
return(minedTwits);
}
numberOfTweetsProcessed += deserializeObject.search_metadata.count.Value;
max_id = deserializeObject.statuses.Last().id;
minedTwits.Add(twitterResult);
}
return(minedTwits);
}
private string GetQueryResponse(ITwitterAuthentificationResponse oAuthCredentials, string query)
{
string json;
HttpWebRequest apiRequest = (HttpWebRequest)WebRequest.Create(string.Concat(TwitterSearchUrl, query));
var authorizationData = $"{oAuthCredentials.TokenType} {oAuthCredentials.AccessToken}";
apiRequest.Headers.Add("Authorization", authorizationData);
apiRequest.Method = "Get";
using (WebResponse oAuthResponse = apiRequest.GetResponse())
{
using (var reader = new StreamReader(oAuthResponse.GetResponseStream()))
{
json = reader.ReadToEnd();
}
}
return(json);
}
public T SearchByQueryDeserialized(ITwitterAuthentificationResponse oAuthCredentials, string query)
{
var json = GetQueryResponse(oAuthCredentials, query);
return(JsonConvert.DeserializeObject <T>(json));
}
public string SearchByQuery(ITwitterAuthentificationResponse oAuthCredentials, string query)
{
var json = GetQueryResponse(oAuthCredentials, query);
return(json);
}
public IList <T> MineTweetsDeserialized(ITwitterAuthentificationResponse twitterAuthentificationResponse, ITwitterMinerQuery minerQuery, long numberOfTweets, long?sinceId = null)
{
return
(MineTweets(twitterAuthentificationResponse, minerQuery, numberOfTweets)
.Select(JsonConvert.DeserializeObject <T>).ToList());
}
private void ProcessData(ITwitterAuthentificationResponse twitterAuthentificationResponse, ITwitterMinerQuery minerQuery,
long numberOfTweets, Action <string> jsonProcessor, long?sinceId, bool runAsync)
{
Stopwatch stopWatch = new Stopwatch();
long max_id = -1;
long numberOfTweetsProcessed = 0;
int numberOfCalls = 0;
stopWatch.Start();
while (numberOfTweetsProcessed < numberOfTweets)
{
string twitterResult;
if (max_id <= 0)
{
if (!sinceId.HasValue)
{
twitterResult = SearchByQuery(twitterAuthentificationResponse, minerQuery.Build());
}
else
{
var query = minerQuery.WithSinceId(sinceId.Value).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
}
else
{
if (!sinceId.HasValue)
{
var query = minerQuery.WithMaxId(max_id - 1).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
else
{
var query = minerQuery.WithMaxId(max_id - 1).WithSinceId(sinceId.Value).Build();
twitterResult = SearchByQuery(twitterAuthentificationResponse, query);
}
}
numberOfCalls++;
dynamic deserializeObject = JsonConvert.DeserializeObject(twitterResult);
if (deserializeObject.statuses.Count == 0)
{
break;
}
numberOfTweetsProcessed += deserializeObject.search_metadata.count.Value;
max_id = deserializeObject.statuses.Last.id.Value;
if (runAsync)
{
Task.Factory.StartNew(() => jsonProcessor(twitterResult));
}
else
{
jsonProcessor(twitterResult);
}
if (numberOfCalls >= 450)
{
stopWatch.Stop();
long elapsedMilliseconds = 15L * 1000L * 60L - stopWatch.ElapsedMilliseconds;
Console.WriteLine("going to sleep...");
Thread.Sleep((int)(elapsedMilliseconds >= 0L?elapsedMilliseconds:0L));
numberOfCalls = 0;
stopWatch.Restart();
}
}
}
public void MineTweetsWithAsyncProcessor(ITwitterAuthentificationResponse twitterAuthentificationResponse, ITwitterMinerQuery minerQuery, long numberOfTweets, Action <string> jsonProcessor, long?since_id = null)
{
ProcessData(twitterAuthentificationResponse, minerQuery, numberOfTweets, jsonProcessor, since_id, true);
}