/// <summary> /// Запуск основной работы /// </summary> public void Run() { // Получаем кучу линков и валидируем их ILinksExtractor links_extractor = LinksTools.CreateExtractor( mOptions.ListFilePath, mOptions.Encoding); List <LinkInfo> links = links_extractor.ExtractLinks(); LinksTools.DeleteEmptyLinks(links); LinksTools.FillEmptyCaptions(links); if (links.Count() == 0) { throw new LogicError("Links is not found."); } // Настраиваем закачиватель и закачиваем им линки ParallelDownloader downloader = new ParallelDownloader( mOptions.FolderPath, mOptions.MaxParallel, mOptions.UpdateInfoMsec, mOptions.IsNumerateFiles ); downloader.Add(links); downloader.Run(); }
public BusLinksExtractor(ILinksExtractor linksExtractor, IEnumerable <HtmlNode> table, List <BusLink> busLinkList) { _linksExtractor = linksExtractor; _linksExtractor.MainPageExtract(table, busLinkList); }
public Crawler(ILogger <ICrawler> logger, ILinksExtractor extractor, IQueryProcessor queryProcessor) { _queryProcessor = queryProcessor; _linksExtractor = extractor; _logger = logger; }
public PageProcessor(string domain, IHttpClientFactory clientFactory, ILinksExtractor linksExtractor) { _domain = domain; _linksExtrator = linksExtractor; _clientFactory = clientFactory; }
public PageProcessor(string domain, IClientWrapper httpClient, ILinksExtractor linksExtractor) : base(httpClient) { _domain = domain; _linksExtrator = linksExtractor; }