public WebCrawler(WebCrawlJob crawlSettings, IContentIndexer <T> indexer, Global <T> .StatusCallBack statusCallback = null)
{
_crawlSettings = crawlSettings;
_logger = SearchFactory.Logger;
_Indexer = indexer;
_pageScrapper = new PageScrapper();
_statusCallback = statusCallback;
}
//TODO: move this into admin
public void ScrapCards()
{
Get["/scrap"] = o =>
{
try
{
logger.Info("GET /scrap invoked");
var exclude = cardsRepository.GetCards().Select(x => x.Name).ToArray();
var cards = new PageScrapper().ScrapCards(exclude).ToList();
SaveImage(cards);
cardsRepository.SaveScrapedCards(cards);
return(Response.AsJson(new { Message = $"Scrapped and saved {cards.Count} cards" }));
}
catch (Exception e)
{
logger.Error(e, "Scrapping failed. Message: {0}", e.Message);
return(Response.AsJson(new { Message = "Scrapping failed. Server error." },
HttpStatusCode.InternalServerError));
}
};
}
