public WebCrawler(WebCrawlJob crawlSettings, IContentIndexer <T> indexer, Global <T> .StatusCallBack statusCallback = null) { _crawlSettings = crawlSettings; _logger = SearchFactory.Logger; _Indexer = indexer; _pageScrapper = new PageScrapper(); _statusCallback = statusCallback; }
//TODO: move this into admin public void ScrapCards() { Get["/scrap"] = o => { try { logger.Info("GET /scrap invoked"); var exclude = cardsRepository.GetCards().Select(x => x.Name).ToArray(); var cards = new PageScrapper().ScrapCards(exclude).ToList(); SaveImage(cards); cardsRepository.SaveScrapedCards(cards); return(Response.AsJson(new { Message = $"Scrapped and saved {cards.Count} cards" })); } catch (Exception e) { logger.Error(e, "Scrapping failed. Message: {0}", e.Message); return(Response.AsJson(new { Message = "Scrapping failed. Server error." }, HttpStatusCode.InternalServerError)); } }; }