public IEnumerable <ParsingRulesData> GetParsingRules(DownloadedContentData data)
{
var settings = ServiceLocator.Resolve <ICrawlerSettingsRepository>();
var rules = settings.GetParsingRules(data.Job);
foreach (var rule in rules)
{
yield return(new ParsingRulesData(data.Job, rule, data.Content));
}
}
public void Should_call_pipeline_methods()
{
var runner = new Runner(); // it will initialize ServiceLocator
#region mocks configuration
#region mock returns
var jobItem = new JobItem();
var downloadedContentData = new DownloadedContentData(jobItem, string.Empty);
var crawlRule = new CrawlRule();
var parsingRulesDatas = new[] {
new ParsingRulesData(jobItem, crawlRule, string.Empty)
};
var parsedContentDatas = new[] {
new ParsedContentData(jobItem, DataBlockType.Link, string.Empty)
};
var crawlRules = new[] {
crawlRule
};
var urlItems = new[] {
new UrlItem()
};
#endregion
#region pipeline mock
var mockPipeline = new Mock <IPipeline>();
mockPipeline.Setup(m => m.DownloadContent(It.IsAny <JobItem>())).Returns(downloadedContentData);
mockPipeline.Setup(m => m.GetParsingRules(It.IsAny <DownloadedContentData>())).Returns(parsingRulesDatas);
mockPipeline.Setup(m => m.ParseContent(It.IsAny <ParsingRulesData>())).Returns(parsedContentDatas);
mockPipeline.Setup(m => m.StoreData(It.IsAny <ParsedContentData>())).Returns(jobItem);
ServiceLocator.RegisterForDependency(mockPipeline.Object);
#endregion
#region frontier mock
// at least one URL should be exist to allow the downloading of content
var mockFrontier = new Mock <IUrlFrontierRepository>();
mockFrontier.Setup(m => m.GetAvailableUrls(It.IsAny <int>(), It.IsAny <DateTime>())).Returns(urlItems);
ServiceLocator.RegisterForDependency(mockFrontier.Object);
#endregion
#region jobs mock
// job should be created for the URL
var mockJob = new Mock <IJobRepository>();
mockJob.Setup(m => m.Start(It.IsAny <UrlItem>())).Returns(jobItem);
ServiceLocator.RegisterForDependency(mockJob.Object);
#endregion
#region settings mock
// at least one rule should be exist to allow the downloading of content
var mockSettings = new Mock <ICrawlerSettingsRepository>();
mockSettings.Setup(m => m.GetParsingRules(It.IsAny <JobItem>())).Returns(crawlRules);
ServiceLocator.RegisterForDependency(mockSettings.Object);
#endregion
#endregion
runner.Run();
mockPipeline.Verify(m => m.DownloadContent(It.IsAny <JobItem>()), Times.Once);
mockPipeline.Verify(m => m.GetParsingRules(It.IsAny <DownloadedContentData>()), Times.Once);
mockPipeline.Verify(m => m.ParseContent(It.IsAny <ParsingRulesData>()), Times.Once);
mockPipeline.Verify(m => m.StoreData(It.IsAny <ParsedContentData>()), Times.Once);
}
