/// <summary>
/// Запуск основной работы
/// </summary>
public void Run()
{
// Получаем кучу линков и валидируем их
ILinksExtractor links_extractor = LinksTools.CreateExtractor(
mOptions.ListFilePath, mOptions.Encoding);
List <LinkInfo> links = links_extractor.ExtractLinks();
LinksTools.DeleteEmptyLinks(links);
LinksTools.FillEmptyCaptions(links);
if (links.Count() == 0)
{
throw new LogicError("Links is not found.");
}
// Настраиваем закачиватель и закачиваем им линки
ParallelDownloader downloader = new ParallelDownloader(
mOptions.FolderPath,
mOptions.MaxParallel,
mOptions.UpdateInfoMsec,
mOptions.IsNumerateFiles
);
downloader.Add(links);
downloader.Run();
}
public BusLinksExtractor(ILinksExtractor linksExtractor, IEnumerable <HtmlNode> table, List <BusLink> busLinkList)
{
_linksExtractor = linksExtractor;
_linksExtractor.MainPageExtract(table, busLinkList);
}
public Crawler(ILogger <ICrawler> logger, ILinksExtractor extractor, IQueryProcessor queryProcessor)
{
_queryProcessor = queryProcessor;
_linksExtractor = extractor;
_logger = logger;
}
public PageProcessor(string domain, IHttpClientFactory clientFactory, ILinksExtractor linksExtractor)
{
_domain = domain;
_linksExtrator = linksExtractor;
_clientFactory = clientFactory;
}
public PageProcessor(string domain, IClientWrapper httpClient, ILinksExtractor linksExtractor) : base(httpClient)
{
_domain = domain;
_linksExtrator = linksExtractor;
}
