public async Task <CrawlerRun> RunAllCrawlers()
{
var watch = new System.Diagnostics.Stopwatch();
watch.Start();
var crawlerRunResult = new CrawlerRun();
if (_config.Crawler == null)
{
return(crawlerRunResult);
}
// Feeds
if (_config.Crawler.FeedsToCrawl.Any())
{
foreach (var feed in _config.Crawler.FeedsToCrawl)
{
var feedCrawler = new FeedCrawler();
var feedResult = await feedCrawler.DoWorkAsync(feed);
crawlerRunResult.Results.Add(feedResult);
}
}
// Tweets
if (_config.Secrets.IsTwitterConsumerConfigured)
{
ITwitterOAuthTokenService oAuthTokenLoader = new TwitterOAuthTokenService();
var oauth = await oAuthTokenLoader.GetAsync(_config.Secrets.TwitterConsumerKey, _config.Secrets.TwitterConsumerSecret);
if (string.IsNullOrWhiteSpace(oauth) == false)
{
if (_config.Crawler.TwitterTimelinesToCrawl.Any())
{
foreach (var handle in _config.Crawler.TwitterTimelinesToCrawl)
{
var twitterTimelineCrawler = new TwitterTimelineCrawler();
twitterTimelineCrawler.OAuthToken = oauth;
var twitterTimelineResult = await twitterTimelineCrawler.DoWorkAsync(handle);
crawlerRunResult.Results.Add(twitterTimelineResult);
}
}
if (_config.Crawler.TwitterUsersToCrawl.Any())
{
foreach (var handle in _config.Crawler.TwitterUsersToCrawl)
{
var twitterUserCrawler = new TwitterUserCrawler();
twitterUserCrawler.OAuthToken = oauth;
var twitterUserResult = await twitterUserCrawler.DoWorkAsync(handle);
crawlerRunResult.Results.Add(twitterUserResult);
}
}
}
}
watch.Stop();
crawlerRunResult.RunDurationInMilliseconds = watch.ElapsedMilliseconds;
crawlerRunResult.RunOn = DateTime.UtcNow;
return(crawlerRunResult);
}
/// <summary>
/// Will run through all applied crawlers.
/// </summary>
public async Task <CrawlerRun> RunAllCrawlers()
{
Trace.TraceInformation($"{nameof(RunAllCrawlers)} invoked.");
var watch = new Stopwatch();
watch.Start();
var crawlerRunResult = new CrawlerRun();
if (_config.Crawler == null)
{
return(crawlerRunResult);
}
// Feeds
if (_config.Crawler.FeedsToCrawl.Any())
{
foreach (var feedConfig in _config.Crawler.FeedsToCrawl)
{
var feedCrawler = new FeedCrawler();
var feedResult = await feedCrawler.DoWorkAsync(feedConfig);
crawlerRunResult.AddResultDataPair(feedConfig.Key, feedResult);
}
}
// GitHubEvents
if (_config.Crawler.GitHubEventsToCrawl.Any())
{
foreach (var githubEventConfig in _config.Crawler.GitHubEventsToCrawl)
{
var eventCrawler = new GitHubEventCrawler();
var eventResult = await eventCrawler.DoWorkAsync(githubEventConfig);
crawlerRunResult.AddResultDataPair(githubEventConfig.Key, eventResult);
}
}
// GitHubIssues
if (_config.Crawler.GitHubIssuesToCrawl.Any())
{
foreach (var githubIssueConfig in _config.Crawler.GitHubIssuesToCrawl)
{
var issueCrawler = new GitHubIssueCrawler();
var issueResult = await issueCrawler.DoWorkAsync(githubIssueConfig);
crawlerRunResult.AddResultDataPair(githubIssueConfig.Key, issueResult);
}
}
// Tweets
if (_config.Secrets.IsTwitterConsumerConfigured)
{
ITwitterOAuthTokenService oAuthTokenLoader = new TwitterOAuthTokenService();
var oauth = await oAuthTokenLoader.GetAsync(_config.Secrets.TwitterConsumerKey, _config.Secrets.TwitterConsumerSecret);
if (string.IsNullOrWhiteSpace(oauth) == false)
{
if (_config.Crawler.TwitterTimelinesToCrawl.Any())
{
foreach (var twitterConfig in _config.Crawler.TwitterTimelinesToCrawl)
{
var twitterTimelineCrawler = new TwitterTimelineCrawler {
OAuthToken = oauth
};
var twitterTimelineResult = await twitterTimelineCrawler.DoWorkAsync(twitterConfig);
crawlerRunResult.AddResultDataPair(twitterConfig.Key, twitterTimelineResult);
}
}
if (_config.Crawler.TwitterUsersToCrawl.Any())
{
foreach (var twitterConfig in _config.Crawler.TwitterUsersToCrawl)
{
var twitterUserCrawler = new TwitterUserCrawler {
OAuthToken = oauth
};
var twitterUserResult = await twitterUserCrawler.DoWorkAsync(twitterConfig);
crawlerRunResult.AddResultDataPair(twitterConfig.Key, twitterUserResult);
}
}
}
}
watch.Stop();
crawlerRunResult.RunDurationInMilliseconds = watch.ElapsedMilliseconds;
crawlerRunResult.RunOn = DateTime.UtcNow;
Trace.TraceInformation($"{nameof(RunAllCrawlers)} done.");
return(crawlerRunResult);
}