Beispiel #1
0
        public DownloadManager(IDownloader downloader, ILinkDataStorage linkDataStorage, CrawlerSettings settings)
        {
            _downloader      = downloader;
            _linkDataStorage = linkDataStorage;
            _settings        = settings;

            InitDownloaderProcess();
        }
Beispiel #2
0
 public WebDownloader(CrawlerSettings settings, IWebContentExtractor contentExtractor)
 {
     _settings         = settings;
     _contentExtractor = contentExtractor;
     _proxy            = new Lazy <IWebProxy>(() =>
     {
         var defaultProxy         = WebRequest.GetSystemWebProxy();
         defaultProxy.Credentials = CredentialCache.DefaultNetworkCredentials;
         return(defaultProxy);
     });
     _cookieContainer = new CookieContainer();
 }