protected void CrawlPage(PageToCrawl page)
{
if (page == null)
{
return;
}
try
{
CQ document = _webRequestManager.MakeRequest(page.PageUrl);
ProccessDocument(document, page);
}
catch (Exception exception)
{
//will log the error for monitoring
_queueManager.Clear();
_threadCordinator.CancelAll();
throw new Exception(exception.Message, exception);
}
}