Beispiel #1
0
 public WebsiteModel() 
 {
     LastOpenConfig = 1;
     // vytvořit instance subobjektů- pro případ nové website
     CrawlerConfig = new CrawlerModel();
     AnalyzerConfig = new AnalyzerModel();
     OrganizerConfig = new OrganizerModel();
     ExportConfig = new ExportModel();
 }
Beispiel #2
0
        /// <summary>
        /// Creates object of type DefaultCrawler needed for Crawler process
        /// </summary>
        /// <param name="crawlerConfig">Crawler configuration</param>
        /// <returns>
        /// Crawler
        /// </returns>
        public DefaultCrawler ConfigureCrawler(CrawlerModel crawlerConfig)
        {
            var crawlerCfg = new CrawlerConfiguration
            {
                Uri = new Uri(crawlerConfig.SiteUrl),
                CountLimit = crawlerConfig.CountLimit,
                DepthLimit = crawlerConfig.DepthLimit,
                IgnoredPrefixes = crawlerConfig.IgnoredPrefixes,
                IgnoredPaths = crawlerConfig.IgnoredPaths
            };

            var directory = crawlerConfig.Directory;
            var imageHandler = new ImageHandler(directory, null);
            var scriptHandler = new ScriptHandler(directory, imageHandler);
            var loader = new HtmlAgilityPackLoader(scriptHandler);

            return new DefaultCrawler(crawlerCfg, loader);
        }