//after page is crawled, here's the result
void crawler_ProcessPageCrawlCompleted(object sender, PageCrawlCompletedArgs e)
{
CrawledPage crawledPage = e.CrawledPage;
if (crawledPage.WebException != null || crawledPage.HttpWebResponse.StatusCode != HttpStatusCode.OK)
{
Console.WriteLine("Crawl of page failed {0}", crawledPage.Uri.AbsoluteUri);
}
else
{
Console.WriteLine("Crawl of page succeeded {0}", crawledPage.Uri.AbsoluteUri);
}
if (string.IsNullOrEmpty(crawledPage.Content.Text))
{
Console.WriteLine("Page had no content {0}", crawledPage.Uri.AbsoluteUri);
}
_dataFinder.Run(crawledPage);
}