private async Task HandleScrapingRequest(ScrapingRequest scrapingRequest, IScraper scraper) { try { var scrapingResult = await scraper.Scrape(new ScrapingContext { ScrapingOrchestrator = this, ScrapingRequest = scrapingRequest, Html = await _downloadManager.Download(scrapingRequest.Url, scraper.WebsiteEncoding) }); if (scrapingResult == null) { // No response to handle (scraper doesn't return a ScrapingResult) return; } if (scrapingResult.Exception != null) { _logger.LogError(scrapingResult.Exception.Message, scrapingResult.Exception); return; } await SaveScrapingResult(scrapingResult); } catch (Exception e) { _logger.LogError($"{e.GetType()}: {e.Message}\n{e.StackTrace}", e); } }
public async Task <ActionResult <ScrapingResponse> > Scrape(ScrapingRequest request) { ScrapingResponse response = new ScrapingResponse { Success = true }; try { // Scrape server IMenuParser menuParser = new BillundPizzaMenuParser(); IMenu menu = menuParser.ParseMenu(); ICollection <MenuItemModel> scraped = menu.Items.Select(x => new MenuItemModel(x)).ToList(); // Load all existing entries ICollection <MenuItemModel> existing = await this._context.MenuItemModel.ToListAsync(); ICollection <MenuCategoryModel> categories = await this._context.MenuCategoryModel.ToListAsync(); // Update existing records with new info foreach (MenuItemModel menuItem in scraped) { MenuItemModel existingItem = existing.FirstOrDefault(x => x.Number == menuItem.Number && x.Name == menuItem.Name && x.Category.Name == menuItem.Category.Name && x.Description == menuItem.Description); if (existingItem != null) { // Entry found, remove from cached collection existing.Remove(existingItem); // Only update database if data is different if (existingItem.Update(menuItem)) { this._context.Entry(existingItem).State = EntityState.Modified; response.NumberUpdated++; } } else if (request.AddNew) { // Add new items from scrape MenuCategoryModel category = categories.FirstOrDefault(x => x.Name == menuItem.Category.Name); if (category == null) { category = new MenuCategoryModel { Name = menuItem.Category.Name }; categories.Add(category); this._context.Add(category); } menuItem.Category = category; this._context.Add(menuItem); response.NumberCreated++; } } // If requested, remove all existing entries not accounted for if (request.RemoveMissing) { foreach (MenuItemModel existingItem in existing) { this._context.Remove(existingItem); response.NumberRemoved++; } } // Save changes await this._context.SaveChangesAsync(); } catch { response.Success = false; } return(response); }