Example #1
0
 public Crawler(CrawlerOptions options)
 {
     _options = options;
     pageVisitedURLMapping = new ConcurrentDictionary <Uri, Uri>();
     _watch = new Stopwatch();
     logger = LogManager.GetCurrentClassLogger();
     pageNotFoundMapping = new ConcurrentDictionary <Uri, PageInfoToExcel>();
 }
Example #2
0
 public Crawler(CrawlerOptions options)
 {
     _options = options;
     pagesVisited = new HashSet<Uri>();
     pagesToVisit = new List<Uri>();
     _watch = new Stopwatch();
     logger = LogManager.GetCurrentClassLogger();
     FirefoxProfile profile = new FirefoxProfile();
     profile.SetPreference("general.useragent.override", _options.UserAgent);
     _driver = new FirefoxDriver(profile);
     //var chromeOptions = new ChromeOptions();
     //chromeOptions.AddArgument("--user-agent=" + _options.UserAgent);
     //_driver = new ChromeDriver(chromeOptions);
 }
Example #3
0
        public Crawler(CrawlerOptions options)
        {
            _options     = options;
            pagesVisited = new HashSet <Uri>();
            pagesToVisit = new List <Uri>();
            _watch       = new Stopwatch();
            logger       = LogManager.GetCurrentClassLogger();
            FirefoxProfile profile = new FirefoxProfile();

            profile.SetPreference("general.useragent.override", _options.UserAgent);
            _driver = new FirefoxDriver(profile);
            //var chromeOptions = new ChromeOptions();
            //chromeOptions.AddArgument("--user-agent=" + _options.UserAgent);
            //_driver = new ChromeDriver(chromeOptions);
        }