public async Task <FeedResult> InvokeRssWithSocialLinksEnabled(int twitterCounts = 0, int facebookShares = 0, string feed = "http://rss.slashdot.org/Slashdot/slashdot", int truncateAt = 0)
{
var sut = new FeedCrawler();
if (feed != string.Empty)
{
string responseData = TestHelperForCurrentProject.GetTestFileContent(TestHelperForCurrentProject.GetTestFilePath(samplesDirectory, slashdotRssSample));
var messageResponse = FakeHttpMessageHandler.GetStringHttpResponseMessage(responseData);
var fakeMessageHandler = new FakeHttpMessageHandler(new HttpMessageOptions()
{
HttpResponseMessage = messageResponse, RequestUri = new Uri(feed)
});
var facebokLoaderMock = A.Fake <IFacebookShareCountLoader>();
A.CallTo(() => facebokLoaderMock.GetAsync(string.Empty)).WithAnyArguments().Returns(facebookShares);
sut = new FeedCrawler(fakeMessageHandler, facebokLoaderMock);
}
var config = new FeedCrawlerConfig
{
Url = feed,
LoadSocialLinkCounters = true,
IncludeRawContent = true,
SummaryTruncateAt = truncateAt
};
return(await sut.DoWorkAsync(config));
}
public async Task <FeedResult> InvokeRssWithCategories(List <string> categories, string feed = "https://blogs.msdn.microsoft.com/webdev/feed/", int truncateAt = 0)
{
var sut = new FeedCrawler();
if (feed != string.Empty)
{
string responseData = TestHelperForCurrentProject.GetTestFileContent(TestHelperForCurrentProject.GetTestFilePath(samplesDirectory, msWebDevRssSample));
var messageResponse = FakeHttpMessageHandler.GetStringHttpResponseMessage(responseData);
var fakeMessageHandler = new FakeHttpMessageHandler(new HttpMessageOptions()
{
HttpResponseMessage = messageResponse, RequestUri = new Uri(feed)
});
var facebokLoaderMock = A.Fake <IFacebookShareCountLoader>();
sut = new FeedCrawler(fakeMessageHandler, facebokLoaderMock);
}
var config = new FeedCrawlerConfig
{
Url = feed,
LoadSocialLinkCounters = false,
IncludeRawContent = true,
SummaryTruncateAt = truncateAt,
FilterByCategories = categories
};
return(await sut.DoWorkAsync(config));
}
public async Task Crawler_Should_Not_Reach_Out_To_Facebook_If_Disabled()
{
string responseData = TestHelperForCurrentProject.GetTestFileContent(TestHelperForCurrentProject.GetTestFilePath(samplesDirectory, gitHubAtomSample));
var messageResponse = FakeHttpMessageHandler.GetStringHttpResponseMessage(responseData);
var fakeMessageHandler = new FakeHttpMessageHandler(new HttpMessageOptions()
{
HttpResponseMessage = messageResponse, RequestUri = new Uri("https://github.com/robertmuehsig.atom")
});
var facebokLoaderMock = A.Fake <IFacebookShareCountLoader>();
var sut = new FeedCrawler(fakeMessageHandler, facebokLoaderMock);
var config = new FeedCrawlerConfig
{
Url = "https://github.com/robertmuehsig.atom",
LoadSocialLinkCounters = false
};
var result = await sut.DoWorkAsync(config);
A.CallTo(() => facebokLoaderMock.GetAsync(string.Empty)).WithAnyArguments().MustNotHaveHappened();
}
public async Task <FeedResult> InvokeAtomSut(int twitterCounts = 0, int facebookShares = 0, string feed = "https://github.com/robertmuehsig.atom", int truncateAt = 0)
{
var atomXmlFile = gitHubAtomSample;
if (feed == "https://blog.nuget.org/feeds/atom.xml")
{
atomXmlFile = nugetBlogAtomSample;
}
string responseData = TestHelperForCurrentProject.GetTestFileContent(TestHelperForCurrentProject.GetTestFilePath(samplesDirectory, atomXmlFile));
var messageResponse = FakeHttpMessageHandler.GetStringHttpResponseMessage(responseData);
var fakeMessageHandler = new FakeHttpMessageHandler(new HttpMessageOptions()
{
HttpResponseMessage = messageResponse, RequestUri = new Uri(feed)
});
var facebokLoaderMock = A.Fake <IFacebookShareCountLoader>();
A.CallTo(() => facebokLoaderMock.GetAsync(string.Empty)).WithAnyArguments().Returns(facebookShares);
var sut = new FeedCrawler(fakeMessageHandler, facebokLoaderMock);
var config = new FeedCrawlerConfig
{
Url = feed,
SummaryTruncateAt = truncateAt
};
return(await sut.DoWorkAsync(config));
}
public async Task Crawler_Can_Read_Media_Thumbnail()
{
string responseData = TestHelperForCurrentProject.GetTestFileContent(TestHelperForCurrentProject.GetTestFilePath(samplesDirectory, youtubeWithMedia));
string feed = "https://www.youtube.com/feeds/videos.xml?channel_id=UCqE3og-pkA7DO90lqNHAR5Q";
var messageResponse = FakeHttpMessageHandler.GetStringHttpResponseMessage(responseData);
var fakeMessageHandler = new FakeHttpMessageHandler(new HttpMessageOptions()
{
HttpResponseMessage = messageResponse, RequestUri = new Uri(feed)
});
var facebokLoaderMock = A.Fake <IFacebookShareCountLoader>();
var sut = new FeedCrawler(fakeMessageHandler, facebokLoaderMock);
var config = new FeedCrawlerConfig
{
Url = feed,
LoadSocialLinkCounters = false,
IncludeRawContent = true,
SummaryTruncateAt = 100
};
var result = await sut.DoWorkAsync(config);
Assert.True(result.FeedItems.Count == 6);
Assert.True(result.FeedItems[0].Thumbnail == "https://i1.ytimg.com/vi/dNaIOGQj15M/hqdefault.jpg");
Assert.True(result.FeedItems[5].Thumbnail == "https://i3.ytimg.com/vi/27yknlB8xeg/hqdefault.jpg");
}
public async Task <FeedCrawlerResult> InvokeSlashdotSutWithSocialLinksEnabled(int twitterCounts = 0, int facebookShares = 0, string feed = "http://rss.slashdot.org/Slashdot/slashdot")
{
var feedLoaderMock = A.Fake <ISyndicationFeedAbstraction>();
var staticFeed = SyndicationFeed.Load(new XmlTextReader(TestHelperForCurrentProject.GetTestFilePath(samplesDirectory, slashdotRssSample)));
A.CallTo(() => feedLoaderMock.Get("http://rss.slashdot.org/Slashdot/slashdot")).Returns(staticFeed);
var twitterLoaderMock = A.Fake <ITwitterTweetCountLoader>();
A.CallTo(() => twitterLoaderMock.GetAsync(string.Empty)).WithAnyArguments().Returns(twitterCounts);
var facebokLoaderMock = A.Fake <IFacebookShareCountLoader>();
A.CallTo(() => facebokLoaderMock.GetAsync(string.Empty)).WithAnyArguments().Returns(facebookShares);
var sut = new FeedCrawler(feedLoaderMock, twitterLoaderMock, facebokLoaderMock);
var config = new FeedCrawlerConfig();
config.Url = feed;
config.LoadSocialLinkCounters = true;
return(await sut.DoWorkAsync(config));
}
public async Task <FeedCrawlerResult> InvokeGitHubSut(int twitterCounts = 0, int facebookShares = 0, string feed = "https://github.com/robertmuehsig.atom")
{
var feedLoaderMock = A.Fake <ISyndicationFeedAbstraction>();
var staticFeed = SyndicationFeed.Load(new XmlTextReader(TestHelperForCurrentProject.GetTestFilePath(samplesDirectory, gitHubAtomSample)));
A.CallTo(() => feedLoaderMock.Get("https://github.com/robertmuehsig.atom")).Returns(staticFeed);
var twitterLoaderMock = A.Fake <ITwitterTweetCountLoader>();
A.CallTo(() => twitterLoaderMock.GetAsync(string.Empty)).WithAnyArguments().Returns(twitterCounts);
var facebokLoaderMock = A.Fake <IFacebookShareCountLoader>();
A.CallTo(() => facebokLoaderMock.GetAsync(string.Empty)).WithAnyArguments().Returns(facebookShares);
var sut = new FeedCrawler(feedLoaderMock, twitterLoaderMock, facebokLoaderMock);
var config = new FeedCrawlerConfig();
config.Url = feed;
return(await sut.DoWorkAsync(config));
}
public async Task Crawler_Should_Not_Reach_Out_To_Twitter_Or_Facebook_If_Disabled()
{
var feedLoaderMock = A.Fake <ISyndicationFeedAbstraction>();
var staticFeed = SyndicationFeed.Load(new XmlTextReader(Path.Combine(samplesDirectory, gitHubAtomSample)));
A.CallTo(() => feedLoaderMock.Get("https://github.com/robertmuehsig.atom")).Returns(staticFeed);
var twitterLoaderMock = A.Fake <ITwitterTweetCountLoader>();
var facebokLoaderMock = A.Fake <IFacebookShareCountLoader>();
var sut = new FeedCrawler(feedLoaderMock, twitterLoaderMock, facebokLoaderMock);
var config = new FeedCrawlerConfig();
config.Url = "https://github.com/robertmuehsig.atom";
config.LoadSocialLinkCounters = false;
var result = await sut.DoWorkAsync(config);
A.CallTo(() => facebokLoaderMock.GetAsync(string.Empty)).WithAnyArguments().MustNotHaveHappened();
A.CallTo(() => twitterLoaderMock.GetAsync(string.Empty)).WithAnyArguments().MustNotHaveHappened();
}
public async Task <CrawlerRun> RunAllCrawlers()
{
var watch = new System.Diagnostics.Stopwatch();
watch.Start();
var crawlerRunResult = new CrawlerRun();
if (_config.Crawler == null)
{
return(crawlerRunResult);
}
// Feeds
if (_config.Crawler.FeedsToCrawl.Any())
{
foreach (var feed in _config.Crawler.FeedsToCrawl)
{
var feedCrawler = new FeedCrawler();
var feedResult = await feedCrawler.DoWorkAsync(feed);
crawlerRunResult.Results.Add(feedResult);
}
}
// Tweets
if (_config.Secrets.IsTwitterConsumerConfigured)
{
ITwitterOAuthTokenService oAuthTokenLoader = new TwitterOAuthTokenService();
var oauth = await oAuthTokenLoader.GetAsync(_config.Secrets.TwitterConsumerKey, _config.Secrets.TwitterConsumerSecret);
if (string.IsNullOrWhiteSpace(oauth) == false)
{
if (_config.Crawler.TwitterTimelinesToCrawl.Any())
{
foreach (var handle in _config.Crawler.TwitterTimelinesToCrawl)
{
var twitterTimelineCrawler = new TwitterTimelineCrawler();
twitterTimelineCrawler.OAuthToken = oauth;
var twitterTimelineResult = await twitterTimelineCrawler.DoWorkAsync(handle);
crawlerRunResult.Results.Add(twitterTimelineResult);
}
}
if (_config.Crawler.TwitterUsersToCrawl.Any())
{
foreach (var handle in _config.Crawler.TwitterUsersToCrawl)
{
var twitterUserCrawler = new TwitterUserCrawler();
twitterUserCrawler.OAuthToken = oauth;
var twitterUserResult = await twitterUserCrawler.DoWorkAsync(handle);
crawlerRunResult.Results.Add(twitterUserResult);
}
}
}
}
watch.Stop();
crawlerRunResult.RunDurationInMilliseconds = watch.ElapsedMilliseconds;
crawlerRunResult.RunOn = DateTime.UtcNow;
return(crawlerRunResult);
}
/// <summary>
/// Will run through all applied crawlers.
/// </summary>
public async Task <CrawlerRun> RunAllCrawlers()
{
Trace.TraceInformation($"{nameof(RunAllCrawlers)} invoked.");
var watch = new Stopwatch();
watch.Start();
var crawlerRunResult = new CrawlerRun();
if (_config.Crawler == null)
{
return(crawlerRunResult);
}
// Feeds
if (_config.Crawler.FeedsToCrawl.Any())
{
foreach (var feedConfig in _config.Crawler.FeedsToCrawl)
{
var feedCrawler = new FeedCrawler();
var feedResult = await feedCrawler.DoWorkAsync(feedConfig);
crawlerRunResult.AddResultDataPair(feedConfig.Key, feedResult);
}
}
// GitHubEvents
if (_config.Crawler.GitHubEventsToCrawl.Any())
{
foreach (var githubEventConfig in _config.Crawler.GitHubEventsToCrawl)
{
var eventCrawler = new GitHubEventCrawler();
var eventResult = await eventCrawler.DoWorkAsync(githubEventConfig);
crawlerRunResult.AddResultDataPair(githubEventConfig.Key, eventResult);
}
}
// GitHubIssues
if (_config.Crawler.GitHubIssuesToCrawl.Any())
{
foreach (var githubIssueConfig in _config.Crawler.GitHubIssuesToCrawl)
{
var issueCrawler = new GitHubIssueCrawler();
var issueResult = await issueCrawler.DoWorkAsync(githubIssueConfig);
crawlerRunResult.AddResultDataPair(githubIssueConfig.Key, issueResult);
}
}
// Tweets
if (_config.Secrets.IsTwitterConsumerConfigured)
{
ITwitterOAuthTokenService oAuthTokenLoader = new TwitterOAuthTokenService();
var oauth = await oAuthTokenLoader.GetAsync(_config.Secrets.TwitterConsumerKey, _config.Secrets.TwitterConsumerSecret);
if (string.IsNullOrWhiteSpace(oauth) == false)
{
if (_config.Crawler.TwitterTimelinesToCrawl.Any())
{
foreach (var twitterConfig in _config.Crawler.TwitterTimelinesToCrawl)
{
var twitterTimelineCrawler = new TwitterTimelineCrawler {
OAuthToken = oauth
};
var twitterTimelineResult = await twitterTimelineCrawler.DoWorkAsync(twitterConfig);
crawlerRunResult.AddResultDataPair(twitterConfig.Key, twitterTimelineResult);
}
}
if (_config.Crawler.TwitterUsersToCrawl.Any())
{
foreach (var twitterConfig in _config.Crawler.TwitterUsersToCrawl)
{
var twitterUserCrawler = new TwitterUserCrawler {
OAuthToken = oauth
};
var twitterUserResult = await twitterUserCrawler.DoWorkAsync(twitterConfig);
crawlerRunResult.AddResultDataPair(twitterConfig.Key, twitterUserResult);
}
}
}
}
watch.Stop();
crawlerRunResult.RunDurationInMilliseconds = watch.ElapsedMilliseconds;
crawlerRunResult.RunOn = DateTime.UtcNow;
Trace.TraceInformation($"{nameof(RunAllCrawlers)} done.");
return(crawlerRunResult);
}
