internal void PrintProgress(HtmlCrawler crawler)
{
if (_timeLastLogged < DateTime.Now.AddSeconds(-10))
{
string template = "Crawled: {0}: {1} pages, {2} links, {3} links remaining";
string progress = string.Format(template, DateTime.Now.ToString("MM/dd/yyyy HH:mm:ss"), crawler.NumberOfPagesCrawled, crawler.NumberOfLinksCrawled, crawler.NumberOfLinksToCrawl);
Console.WriteLine(progress);
_timeLastLogged = DateTime.Now;
}
}
static void Main(string[] args)
{
Console.WriteLine("Started Crawling at " + DateTime.Now.ToString("MM/dd/yyyy HH:mm:ss"));
var crawlerProgress = new HtmlCrawlerProgress();
var crawler = new HtmlCrawler("http://www.leg.wa.gov", "/house", crawlerProgress);
crawler.Crawl(200);
Console.WriteLine("Press the Enter key to exit anytime... ");
Console.ReadLine();
}
