private void crawler_ProcessPageCrawlCompleted(object sender, PageCrawlCompletedArgs e)
{
CrawledPage crawledPage = e.CrawledPage;
if (_amazonHelper.IsCaptchaPage(crawledPage))
{
lock (_captchaLock)
{
InvokeIfRequired(() =>
{
var form = new BrowserForm();
form.Browser.Navigate(crawledPage.Uri);
form.ShowDialog();
});
}
}
var products = _amazonHelper.GetProductsFromDetailPage(crawledPage);
foreach (var p in products)
{
if (!_products.Any(x => x.external_product_id == p.external_product_id))
{
InvokeIfRequired(() =>
{
_products.Add(p);
Application.DoEvents();
});
}
}
if (crawledPage.WebException != null || crawledPage.HttpWebResponse.StatusCode != HttpStatusCode.OK)
{
_logger.InfoFormat("Crawl of page failed {0}", crawledPage.Uri.AbsoluteUri);
}
else
{
_logger.InfoFormat("Crawl of page succeeded {0}", crawledPage.Uri.AbsoluteUri);
}
if (string.IsNullOrEmpty(crawledPage.Content.Text))
{
_logger.InfoFormat("Page had no content {0}", crawledPage.Uri.AbsoluteUri);
}
}
private void ShowBrowserButton_Click(object sender, EventArgs e)
{
var form = new BrowserForm();
form.Show();
}