private void Finish()
{
_driver.Quit();
SaveSitemap();
Save404Pages();
ExcelHandler.DataTableToExcel(_options.FolderPath + "\\PageNonValidateList.xlsx", pagesErrorList);
_watch.Stop();
logger.Info("Finish all task in {0}", _watch.Elapsed);
}
private Dictionary <string, string> pageParentURLMapping; //Key is current Page, Content is parent Page
public Crawler(CrawlerOptions options)
{
_options = options;
pagesVisited = new HashSet <Uri>();
pagesNotFound = new HashSet <Uri>();
pagesToVisit = new List <Uri>();
_watch = new Stopwatch();
pagesErrorList = new DataTable();
pagesErrorList = ExcelHandler.InitTable(pagesErrorList);
logger = LogManager.GetCurrentClassLogger();
pageParentURLMapping = new Dictionary <string, string>();
var chromeOptions = new ChromeOptions();
chromeOptions.AddArgument("--user-agent=" + _options.UserAgent);
_driver = new ChromeDriver(chromeOptions);
}
