/// <summary> /// Starts the <see cref="Crawlers.UK.UKSupremeCourt"/> crawler /// </summary> /// <param name="logger"></param> /// <returns></returns> public static async Task RunUKSupremeCourtAsync(ILog logger) { var http = new Http(); var crawler = new UKSupremeCourt(http, logger); var manager = new HttpCrawlerManager(logger, CrawlerHttpManagerConfig.Default); await manager.StartAsync(crawler); }
/// <summary> /// Starts the <see cref="Crawlers.Eurlex.Eurlex"/> crawler /// </summary> /// <param name="logger"></param> /// <returns></returns> public static async Task RunEurlexAsync(ILog logger) { var http = new Http(Http.DefaultRetryCount, Http.DefaultReUseCookies, allowRedirect: true, Http.DefaultTimeout); var crawler = new Eurlex(http, logger); var manager = new HttpCrawlerManager(logger, CrawlerHttpManagerConfig.Default); await manager.StartAsync(crawler); }
/// <summary> /// Starts crawler from the specified cmd arguments /// </summary> /// <param name="args"> /// Atleast on argument which must represent the name of the crawler to be started (example: Eurlex or Interlex.Crawler.Crawlers.Eurlex.Eurlex). /// Allowed argumnets are: -retrycount, -timeout, -reusecookies, -allowredirect /// </param> /// <param name="logger">Logger</param> /// <returns></returns> public static async Task CreateCmdArgs(string[] args, ILog logger) { var crawler = CreateCrawlerFromCmdArgs(args, logger); if (crawler is BaseHttpCrawler bhc) { var manager = new HttpCrawlerManager(logger, CrawlerHttpManagerConfig.Default); await manager.StartAsync(bhc); } else { await(crawler as BaseGenericCrawler).StartAsync(); } }