public async Task RunAsync(ScrapeSettings settings, CancellationTokenSource cts) { Log.Information("ScrapeSettings = {Settings}", JsonConvert.SerializeObject(settings)); if (settings.RebuildDatabase) { await RebuildDatabaseAsync(cts.Token); } if (settings.UpdateDepartments) { await UpdateDepartmentalSectionsAsync(cts.Token); } if (settings.UpdateCategories) { await UpdateCategoriesAsync(cts.Token); } if (settings.UpdateProducts) { await UpdateProductsAsync(cts.Token); } if (settings.UpdateProductDetails) { await UpdateProductsDetailsAsync(settings, cts.Token); } if (settings.UpdateProductVariances) { await UpdateProductVariancesAsync(settings, cts.Token); } }
public GlobalSettings(IConfiguration config) { Cache = new CacheSettings(config); Database = new DatabaseSettings(config); Redis = new RedisSettings(config); Scraper = new ScrapeSettings(config); LogRequests = config.GetValue("LogRequests", false); }
private async Task UpdateProductVariancesAsync(ScrapeSettings settings, CancellationToken cancellationToken) { using (var dbContext = _serviceProvider.GetService <ApplicationDbContext>()) using (var httpClient = _serviceProvider.GetService <AmazonHttpClient>()) { Log.Information("Updating Product Variances"); var pendingStatus = ProductStatus.Pending.ToString(); var variancesQuery = dbContext.ProductVariances .Where(x => x.Status == pendingStatus); if (settings.EnableProductDetailBatchScraping && settings.BatchSize > -1) { variancesQuery = variancesQuery .OrderByDescending(x => x.UpdatedOn) .Take(settings.BatchSize); } var variances = variancesQuery.ToArray(); var getProductVariances = variances .Select(async p => { try { var details = await httpClient.GetProductDetailAsync(p.Uri, cancellationToken); p.CurrentPrice = details.CurrentPrice; p.OriginalPrice = details.OriginalPrice; p.UpdatedOn = DateTime.Now; p.Status = ProductStatus.Completed.ToString(); } catch (Exception e) { p.Errors = e.Message + e.StackTrace; } lock (Locker) { dbContext.SaveChanges(); } return(p); }) .ToArray(); await Task.WhenAll(getProductVariances); } }
// This method gets called by the runtime. Use this method to add services to the container. public void ConfigureServices(IServiceCollection services) { services.AddMvc(); services.TryAddSingleton <IWebPageDownloader, WebPageDownloader>(); services.AddSingleton <IScrapeSettings>(s => { var scrapeSettings = new ScrapeSettings(new CultureInfo("es-ES"), "http://www.morningstar.es/es/{0}/snapshot/snapshot.aspx?id={1}"); return(scrapeSettings); }); services.AddTransient <IQuoteScraper, EtfQuoteScraper>(); services.AddTransient <IQuoteScraper, MutualFundQuoteScraper>(); }
public ScrapeService(ILogger <ScrapeService> logger, IOptions <ScrapeSettings> scrapeSettings) { _logger = logger; _scrapeSettings = scrapeSettings.Value; }
private async Task UpdateProductsDetailsAsync(ScrapeSettings settings, CancellationToken cancellationToken) { using (var dbContext = _serviceProvider.GetService <ApplicationDbContext>()) using (var httpClient = _serviceProvider.GetService <AmazonHttpClient>()) { Log.Information("Updating Product Details"); var pendingStatus = ProductStatus.Pending.ToString(); var productsQuery = dbContext.Products .Where(x => x.Status == pendingStatus); if (settings.EnableProductDetailBatchScraping && settings.BatchSize > -1) { productsQuery = productsQuery .OrderByDescending(x => x.UpdatedOn) .Take(settings.BatchSize); } var products = productsQuery.ToArray(); var getProductDetailTasks = products .Select(async p => { try { var details = await httpClient.GetProductDetailAsync(p.Uri, cancellationToken); p.Url = details.Url; p.Name = details.Name; p.Brand = details.Brand; p.Asin = details.Asin; p.CurrentPrice = details.CurrentPrice; p.OriginalPrice = details.OriginalPrice; p.Dimensions = details.Dimensions; p.ItemWeight = details.ItemWeight; p.ShippingWeight = details.ShippingWeight; p.Manufacturer = details.Manufacturer; p.ModelNumber = details.ModelNumber; p.Rating = details.Rating; p.TotalReviews = details.TotalReviews; p.FirstAvailableOn = details.FirstAvailableOn; p.Errors = details.Errors; p.Status = ProductStatus.Completed.ToString(); p.UpdatedOn = DateTime.Now; // set ranking if (p.Rank != null) { p.SetRank(details.Rank); } if (details.Variances?.Count > 0) { p.HasVariances = true; p.Variances = details.Variances .Select(x => new ProductVariance { Name = x.Name, Asin = x.Asin, Url = x.Url, Status = ProductStatus.Pending.ToString(), UpdatedOn = DateTime.Now }) .ToList(); } } catch (Exception e) { p.Errors += " " + e.Message + e.StackTrace; } lock (Locker) { dbContext.SaveChanges(); } return(p); }) .ToArray(); await Task.WhenAll(getProductDetailTasks); } }
// This function will get triggered/executed when a new message is written // on an Azure Queue called queue. public async Task ProcessQueueMessage([QueueTrigger("%APPSETTING_AzureQueueName%")] ScrapeSettings settings, TextWriter log) { var cts = new CancellationTokenSource(); await _scrapingService.RunAsync(settings, cts); }