private async Task HandleScrapingRequest(ScrapingRequest scrapingRequest, IScraper scraper)
{
try
{
var scrapingResult = await scraper.Scrape(new ScrapingContext
{
ScrapingOrchestrator = this,
ScrapingRequest = scrapingRequest,
Html = await _downloadManager.Download(scrapingRequest.Url, scraper.WebsiteEncoding)
});
if (scrapingResult == null)
{
// No response to handle (scraper doesn't return a ScrapingResult)
return;
}
if (scrapingResult.Exception != null)
{
_logger.LogError(scrapingResult.Exception.Message, scrapingResult.Exception);
return;
}
await SaveScrapingResult(scrapingResult);
}
catch (Exception e)
{
_logger.LogError($"{e.GetType()}: {e.Message}\n{e.StackTrace}", e);
}
}
public async Task <ActionResult <ScrapingResponse> > Scrape(ScrapingRequest request)
{
ScrapingResponse response = new ScrapingResponse {
Success = true
};
try
{
// Scrape server
IMenuParser menuParser = new BillundPizzaMenuParser();
IMenu menu = menuParser.ParseMenu();
ICollection <MenuItemModel> scraped = menu.Items.Select(x => new MenuItemModel(x)).ToList();
// Load all existing entries
ICollection <MenuItemModel> existing = await this._context.MenuItemModel.ToListAsync();
ICollection <MenuCategoryModel> categories = await this._context.MenuCategoryModel.ToListAsync();
// Update existing records with new info
foreach (MenuItemModel menuItem in scraped)
{
MenuItemModel existingItem = existing.FirstOrDefault(x => x.Number == menuItem.Number && x.Name == menuItem.Name &&
x.Category.Name == menuItem.Category.Name && x.Description == menuItem.Description);
if (existingItem != null)
{
// Entry found, remove from cached collection
existing.Remove(existingItem);
// Only update database if data is different
if (existingItem.Update(menuItem))
{
this._context.Entry(existingItem).State = EntityState.Modified;
response.NumberUpdated++;
}
}
else if (request.AddNew)
{
// Add new items from scrape
MenuCategoryModel category = categories.FirstOrDefault(x => x.Name == menuItem.Category.Name);
if (category == null)
{
category = new MenuCategoryModel {
Name = menuItem.Category.Name
};
categories.Add(category);
this._context.Add(category);
}
menuItem.Category = category;
this._context.Add(menuItem);
response.NumberCreated++;
}
}
// If requested, remove all existing entries not accounted for
if (request.RemoveMissing)
{
foreach (MenuItemModel existingItem in existing)
{
this._context.Remove(existingItem);
response.NumberRemoved++;
}
}
// Save changes
await this._context.SaveChangesAsync();
}
catch
{
response.Success = false;
}
return(response);
}