// Perform the initial robots.txt crawl.
private void Startup()
{
_statusManager.UpdateCrawlrStatus(
"Initializing",
_crawlrData,
_storageManager
);
RobotsTxtCrawlr.CrawlRobotsTxt(ref _crawlrData, ref _storageManager);
_statusManager.UpdateQueueSize(_storageManager, _crawlrData.NumXmlsQueued, _crawlrData.NumUrlsQueued);
}
// Perform the initial robots.txt crawl.
private void Startup()
{
// Update status
_statusManager.UpdateCrawlrStatus(
"Initializing",
_crawlrData,
_storageManager
);
// Crawl robots.txt
RobotsTxtCrawlr.CrawlRobotsTxt(ref _crawlrData, ref _storageManager);
// Initial update of queue size
_statusManager.UpdateQueueSize(_storageManager, _crawlrData.NumXmlsQueued, _crawlrData.NumUrlsQueued);
}
