protected Crawller(DBManager dbManager, CancellationToken cancellationToken)
{
CrawlerProfile =
dbManager.
CrawlingManagerProfileCollection.
Find(profile => profile.Id == GetType().Name).
Single();
TaskExtension.RunPeriodicly(
async() =>
await dbManager.
CrawlingManagerProfileCollection.
FindOneAndReplaceAsync(profile => profile.Id == GetType().Name, CrawlerProfile),
15.Minutes(),
cancellationToken);
}
public CrawlerManager(DBManager dbManager, CancellationToken cancellationToken)
{
_crawlers = new List <Crawller>
{
new ShefLavanCrawler(dbManager, cancellationToken),
new MakoCrawler(dbManager, cancellationToken)
};
_scrapingManager = new ScrapingManager(dbManager);
TaskExtension.RunPeriodicly(
async() =>
{
var urls =
(await _crawlers.
Select(async crawler => await crawler.CrawlAsync()).
ToList()).
SelectMany(list => list).
ToList();
await _scrapingManager.ScrapeAsync(urls);
},
1.Hours(),
cancellationToken);
}
