public static void StartCrawlEbuyer(string url)
{
try
{
PoliteWebCrawler crawler = new PoliteWebCrawler();
crawler.PageCrawlStartingAsync += crawler_ProcessPageCrawlStarting;
crawler.PageCrawlCompletedAsync += crawler_ProcessPageCrawlCompleted;
crawler.PageCrawlDisallowedAsync += crawler_PageCrawlDisallowed;
crawler.PageLinksCrawlDisallowedAsync += crawler_PageLinksCrawlDisallowed;
TimeSpan ts = new TimeSpan(0, 0, 5);
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(ts);
CrawlResult result = crawler.Crawl(new Uri(url), cancellationTokenSource);
if (result.ErrorOccurred)
{
Console.WriteLine("Crawl of {0} completed with error: {1}", result.RootUri.AbsoluteUri, result.ErrorException.Message);
}
else
{
Console.WriteLine("Crawl of {0} completed without error.", result.RootUri.AbsoluteUri);
}
}catch (Exception)
{
}
ExtractingHtml.ExtractDetailsEbuyer();
}
//Crawling code for GSM
public static void StartCrawlGSM(string url)
{
PoliteWebCrawler crawler = new PoliteWebCrawler();
crawler.PageCrawlStartingAsync += crawler_ProcessPageCrawlStartingGSM;
crawler.PageCrawlCompletedAsync += crawler_ProcessPageCrawlCompletedGSM;
crawler.PageCrawlDisallowedAsync += crawler_PageCrawlDisallowedGSM;
crawler.PageLinksCrawlDisallowedAsync += crawler_PageLinksCrawlDisallowedGSM;
TimeSpan ts = new TimeSpan(0, 0, 0);
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(ts);
CrawlResult result = crawler.Crawl(new Uri(url), cancellationTokenSource);
if (result.ErrorOccurred)
{
Console.WriteLine("Crawl of {0} completed with error: {1}", result.RootUri.AbsoluteUri, result.ErrorException.Message);
}
else
{
Console.WriteLine("Crawl of {0} completed without error.", result.RootUri.AbsoluteUri);
}
//FileStream fs = new FileStream("url.txt", FileMode.Open);
//StreamReader sr = new StreamReader(fs);
//string str = "";
//while ((str = sr.ReadLine()) != null)
//{
// StartCrawl(str);
//}
ExtractingHtml.ExtractingDetailsGSM();
}
