public async Task RunAsync(ScrapeSettings settings, CancellationTokenSource cts)
{
Log.Information("ScrapeSettings = {Settings}", JsonConvert.SerializeObject(settings));
if (settings.RebuildDatabase)
{
await RebuildDatabaseAsync(cts.Token);
}
if (settings.UpdateDepartments)
{
await UpdateDepartmentalSectionsAsync(cts.Token);
}
if (settings.UpdateCategories)
{
await UpdateCategoriesAsync(cts.Token);
}
if (settings.UpdateProducts)
{
await UpdateProductsAsync(cts.Token);
}
if (settings.UpdateProductDetails)
{
await UpdateProductsDetailsAsync(settings, cts.Token);
}
if (settings.UpdateProductVariances)
{
await UpdateProductVariancesAsync(settings, cts.Token);
}
}
public GlobalSettings(IConfiguration config)
{
Cache = new CacheSettings(config);
Database = new DatabaseSettings(config);
Redis = new RedisSettings(config);
Scraper = new ScrapeSettings(config);
LogRequests = config.GetValue("LogRequests", false);
}
private async Task UpdateProductVariancesAsync(ScrapeSettings settings, CancellationToken cancellationToken)
{
using (var dbContext = _serviceProvider.GetService <ApplicationDbContext>())
using (var httpClient = _serviceProvider.GetService <AmazonHttpClient>())
{
Log.Information("Updating Product Variances");
var pendingStatus = ProductStatus.Pending.ToString();
var variancesQuery = dbContext.ProductVariances
.Where(x => x.Status == pendingStatus);
if (settings.EnableProductDetailBatchScraping && settings.BatchSize > -1)
{
variancesQuery = variancesQuery
.OrderByDescending(x => x.UpdatedOn)
.Take(settings.BatchSize);
}
var variances = variancesQuery.ToArray();
var getProductVariances = variances
.Select(async p =>
{
try
{
var details = await httpClient.GetProductDetailAsync(p.Uri, cancellationToken);
p.CurrentPrice = details.CurrentPrice;
p.OriginalPrice = details.OriginalPrice;
p.UpdatedOn = DateTime.Now;
p.Status = ProductStatus.Completed.ToString();
}
catch (Exception e)
{
p.Errors = e.Message + e.StackTrace;
}
lock (Locker)
{
dbContext.SaveChanges();
}
return(p);
})
.ToArray();
await Task.WhenAll(getProductVariances);
}
}
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddMvc();
services.TryAddSingleton <IWebPageDownloader, WebPageDownloader>();
services.AddSingleton <IScrapeSettings>(s =>
{
var scrapeSettings = new ScrapeSettings(new CultureInfo("es-ES"),
"http://www.morningstar.es/es/{0}/snapshot/snapshot.aspx?id={1}");
return(scrapeSettings);
});
services.AddTransient <IQuoteScraper, EtfQuoteScraper>();
services.AddTransient <IQuoteScraper, MutualFundQuoteScraper>();
}
public ScrapeService(ILogger <ScrapeService> logger, IOptions <ScrapeSettings> scrapeSettings)
{
_logger = logger;
_scrapeSettings = scrapeSettings.Value;
}
private async Task UpdateProductsDetailsAsync(ScrapeSettings settings, CancellationToken cancellationToken)
{
using (var dbContext = _serviceProvider.GetService <ApplicationDbContext>())
using (var httpClient = _serviceProvider.GetService <AmazonHttpClient>())
{
Log.Information("Updating Product Details");
var pendingStatus = ProductStatus.Pending.ToString();
var productsQuery = dbContext.Products
.Where(x => x.Status == pendingStatus);
if (settings.EnableProductDetailBatchScraping && settings.BatchSize > -1)
{
productsQuery = productsQuery
.OrderByDescending(x => x.UpdatedOn)
.Take(settings.BatchSize);
}
var products = productsQuery.ToArray();
var getProductDetailTasks = products
.Select(async p =>
{
try
{
var details = await httpClient.GetProductDetailAsync(p.Uri, cancellationToken);
p.Url = details.Url;
p.Name = details.Name;
p.Brand = details.Brand;
p.Asin = details.Asin;
p.CurrentPrice = details.CurrentPrice;
p.OriginalPrice = details.OriginalPrice;
p.Dimensions = details.Dimensions;
p.ItemWeight = details.ItemWeight;
p.ShippingWeight = details.ShippingWeight;
p.Manufacturer = details.Manufacturer;
p.ModelNumber = details.ModelNumber;
p.Rating = details.Rating;
p.TotalReviews = details.TotalReviews;
p.FirstAvailableOn = details.FirstAvailableOn;
p.Errors = details.Errors;
p.Status = ProductStatus.Completed.ToString();
p.UpdatedOn = DateTime.Now;
// set ranking
if (p.Rank != null)
{
p.SetRank(details.Rank);
}
if (details.Variances?.Count > 0)
{
p.HasVariances = true;
p.Variances = details.Variances
.Select(x => new ProductVariance
{
Name = x.Name,
Asin = x.Asin,
Url = x.Url,
Status = ProductStatus.Pending.ToString(),
UpdatedOn = DateTime.Now
})
.ToList();
}
}
catch (Exception e)
{
p.Errors += " " + e.Message + e.StackTrace;
}
lock (Locker)
{
dbContext.SaveChanges();
}
return(p);
})
.ToArray();
await Task.WhenAll(getProductDetailTasks);
}
}
// This function will get triggered/executed when a new message is written
// on an Azure Queue called queue.
public async Task ProcessQueueMessage([QueueTrigger("%APPSETTING_AzureQueueName%")] ScrapeSettings settings, TextWriter log)
{
var cts = new CancellationTokenSource();
await _scrapingService.RunAsync(settings, cts);
}
