Exemple #1
0
 protected void CrawlPage(PageToCrawl page)
 {
     if (page == null)
     {
         return;
     }
     try
     {
         CQ document = _webRequestManager.MakeRequest(page.PageUrl);
         ProccessDocument(document, page);
     }
     catch (Exception exception)
     {
         //will log the error for monitoring
         _queueManager.Clear();
         _threadCordinator.CancelAll();
         throw new Exception(exception.Message, exception);
     }
 }