Beispiel #1
0
 // Perform the initial robots.txt crawl.
 private void Startup()
 {
     _statusManager.UpdateCrawlrStatus(
         "Initializing",
         _crawlrData,
         _storageManager
         );
     RobotsTxtCrawlr.CrawlRobotsTxt(ref _crawlrData, ref _storageManager);
     _statusManager.UpdateQueueSize(_storageManager, _crawlrData.NumXmlsQueued, _crawlrData.NumUrlsQueued);
 }
Beispiel #2
0
        // Perform the initial robots.txt crawl.
        private void Startup()
        {
            // Update status
            _statusManager.UpdateCrawlrStatus(
                "Initializing",
                _crawlrData,
                _storageManager
                );

            // Crawl robots.txt
            RobotsTxtCrawlr.CrawlRobotsTxt(ref _crawlrData, ref _storageManager);

            // Initial update of queue size
            _statusManager.UpdateQueueSize(_storageManager, _crawlrData.NumXmlsQueued, _crawlrData.NumUrlsQueued);
        }