public static void Download(IWebsite website) { try { using (var client = new WebClient()) { _website = website; var downloadUrl = client.DownloadString(website.Url); var scraper = new Scraper(); website.ScrapeCriteriaPart = scraper.AddScrapeCriteriaPart(website.AppSettings["ScrapeCriteriaPart"], RegexOptions.Singleline); website.ScrapeCriteria = scraper.AddScrapeCriteria(downloadUrl, website.AppSettings["ScrapeCriteria"], RegexOptions.Singleline, website.ScrapeCriteriaPart); _scrapedElements = scraper.Scrape(website.ScrapeCriteria); ScrapeFileManager.AppendToFile(_scrapedElements); } } catch (Exception e) { Logger.Error(e.Message); throw; } }
public void Run() { try { RunFlatWebsiteScraper("domoPliusAppSettings", new DomoPliusLt()); RunFlatWebsiteScraper("aruodasAppSettings", new AruodasLt()); ScrapeFileManager.ReadFromFile(); } catch (Exception e) { Logger.Error(e.Message); throw; } }