Ejemplo n.º 1
0
        public async Task <CrawlerRun> RunAllCrawlers()
        {
            var watch = new System.Diagnostics.Stopwatch();

            watch.Start();

            var crawlerRunResult = new CrawlerRun();

            if (_config.Crawler == null)
            {
                return(crawlerRunResult);
            }

            // Feeds
            if (_config.Crawler.FeedsToCrawl.Any())
            {
                foreach (var feed in _config.Crawler.FeedsToCrawl)
                {
                    var feedCrawler = new FeedCrawler();
                    var feedResult  = await feedCrawler.DoWorkAsync(feed);

                    crawlerRunResult.Results.Add(feedResult);
                }
            }

            // Tweets
            if (_config.Secrets.IsTwitterConsumerConfigured)
            {
                ITwitterOAuthTokenService oAuthTokenLoader = new TwitterOAuthTokenService();
                var oauth = await oAuthTokenLoader.GetAsync(_config.Secrets.TwitterConsumerKey, _config.Secrets.TwitterConsumerSecret);

                if (string.IsNullOrWhiteSpace(oauth) == false)
                {
                    if (_config.Crawler.TwitterTimelinesToCrawl.Any())
                    {
                        foreach (var handle in _config.Crawler.TwitterTimelinesToCrawl)
                        {
                            var twitterTimelineCrawler = new TwitterTimelineCrawler();
                            twitterTimelineCrawler.OAuthToken = oauth;

                            var twitterTimelineResult = await twitterTimelineCrawler.DoWorkAsync(handle);

                            crawlerRunResult.Results.Add(twitterTimelineResult);
                        }
                    }

                    if (_config.Crawler.TwitterUsersToCrawl.Any())
                    {
                        foreach (var handle in _config.Crawler.TwitterUsersToCrawl)
                        {
                            var twitterUserCrawler = new TwitterUserCrawler();
                            twitterUserCrawler.OAuthToken = oauth;

                            var twitterUserResult = await twitterUserCrawler.DoWorkAsync(handle);

                            crawlerRunResult.Results.Add(twitterUserResult);
                        }
                    }
                }
            }

            watch.Stop();
            crawlerRunResult.RunDurationInMilliseconds = watch.ElapsedMilliseconds;
            crawlerRunResult.RunOn = DateTime.UtcNow;

            return(crawlerRunResult);
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Will run through all applied crawlers.
        /// </summary>
        public async Task <CrawlerRun> RunAllCrawlers()
        {
            Trace.TraceInformation($"{nameof(RunAllCrawlers)} invoked.");
            var watch = new Stopwatch();

            watch.Start();

            var crawlerRunResult = new CrawlerRun();

            if (_config.Crawler == null)
            {
                return(crawlerRunResult);
            }

            // Feeds
            if (_config.Crawler.FeedsToCrawl.Any())
            {
                foreach (var feedConfig in _config.Crawler.FeedsToCrawl)
                {
                    var feedCrawler = new FeedCrawler();
                    var feedResult  = await feedCrawler.DoWorkAsync(feedConfig);

                    crawlerRunResult.AddResultDataPair(feedConfig.Key, feedResult);
                }
            }

            // GitHubEvents
            if (_config.Crawler.GitHubEventsToCrawl.Any())
            {
                foreach (var githubEventConfig in _config.Crawler.GitHubEventsToCrawl)
                {
                    var eventCrawler = new GitHubEventCrawler();
                    var eventResult  = await eventCrawler.DoWorkAsync(githubEventConfig);

                    crawlerRunResult.AddResultDataPair(githubEventConfig.Key, eventResult);
                }
            }

            // GitHubIssues
            if (_config.Crawler.GitHubIssuesToCrawl.Any())
            {
                foreach (var githubIssueConfig in _config.Crawler.GitHubIssuesToCrawl)
                {
                    var issueCrawler = new GitHubIssueCrawler();
                    var issueResult  = await issueCrawler.DoWorkAsync(githubIssueConfig);

                    crawlerRunResult.AddResultDataPair(githubIssueConfig.Key, issueResult);
                }
            }

            // Tweets
            if (_config.Secrets.IsTwitterConsumerConfigured)
            {
                ITwitterOAuthTokenService oAuthTokenLoader = new TwitterOAuthTokenService();
                var oauth = await oAuthTokenLoader.GetAsync(_config.Secrets.TwitterConsumerKey, _config.Secrets.TwitterConsumerSecret);

                if (string.IsNullOrWhiteSpace(oauth) == false)
                {
                    if (_config.Crawler.TwitterTimelinesToCrawl.Any())
                    {
                        foreach (var twitterConfig in _config.Crawler.TwitterTimelinesToCrawl)
                        {
                            var twitterTimelineCrawler = new TwitterTimelineCrawler {
                                OAuthToken = oauth
                            };

                            var twitterTimelineResult = await twitterTimelineCrawler.DoWorkAsync(twitterConfig);

                            crawlerRunResult.AddResultDataPair(twitterConfig.Key, twitterTimelineResult);
                        }
                    }

                    if (_config.Crawler.TwitterUsersToCrawl.Any())
                    {
                        foreach (var twitterConfig in _config.Crawler.TwitterUsersToCrawl)
                        {
                            var twitterUserCrawler = new TwitterUserCrawler {
                                OAuthToken = oauth
                            };

                            var twitterUserResult = await twitterUserCrawler.DoWorkAsync(twitterConfig);

                            crawlerRunResult.AddResultDataPair(twitterConfig.Key, twitterUserResult);
                        }
                    }
                }
            }

            watch.Stop();
            crawlerRunResult.RunDurationInMilliseconds = watch.ElapsedMilliseconds;
            crawlerRunResult.RunOn = DateTime.UtcNow;

            Trace.TraceInformation($"{nameof(RunAllCrawlers)} done.");

            return(crawlerRunResult);
        }