public async Task <CompanyData> InsertCompanyIntoDb(CompanyData company)
{
//Number of pages
company.CompanyState = CompanyState.Processing;
await _dbContext.SaveChangesAsync();
var counter = 1;
var page = await _browsingContextService.OpenPageAsync(company.Url.Replace(".html", $";{counter}.html"));
//Get all pages for current company
while (page.Url != company.Url || counter == 1)
{
var products = CompanyParsingService.ParseCompanyProducts(company, page).ToArray();
company.Products.AddRange(products);
await TaskQueue.QueueBackgroundWorkItemsRangeAsync(products);
_logger.LogInformation($"CompanyService new products of {company.Name} company, data-chunk was added");
counter++;
Thread.Sleep(5000);
page = await _browsingContextService.OpenPageAsync(company.Url.Replace(".html", $";{counter}.html"));
if (page != null)
{
continue;
}
company.CompanyState = CompanyState.Failed;
await _dbContext.SaveChangesAsync();
return(company);
}
company.CompanyState = CompanyState.Success;
company.SyncDate = DateTime.Now;
var jsonCompanyDat = new CompanyJson
{
ExternalId = company.ExternalId,
PortalPageUrl = company.Url,
SyncDate = company.SyncDate,
Name = company.Name
};
var generator = new JSchemaGenerator();
var fullCategorySchema = generator.Generate(typeof(CompanyJson)).ToString();
var fullCategoryJson = JsonConvert.SerializeObject(jsonCompanyDat);
company.JsonData = fullCategoryJson;
company.JsonDataSchema = fullCategorySchema;
await _dbContext.SaveChangesAsync();
return(company);
}
public async Task <ProductData> InsertProductPageIntoDb(ProductData product)
{
var productPage = await _browsingContextService.OpenPageAsync(product.Url);
if (productPage == null)
{
throw new NullReferenceException(nameof(productPage));
}
if (productPage.StatusCode == HttpStatusCode.TooManyRequests)
{
throw new TooManyRequestsException();
}
var parsedProduct = ProductParsingService.ParseSinglePage(product, productPage);
var categories = CategoryParsingService.ParseCategories(product, productPage);
var enumerableCategories = categories as CategoryData[] ?? categories.ToArray();
parsedProduct.Categories = enumerableCategories.ToList();
foreach (var category in enumerableCategories.Where(category =>
_dbContext.Categories.FirstOrDefault(cat => cat.Name == category.Name) == null))
{
if (category.SupCategoryData?.SupCategoryData != null)
{
category.SupCategoryData.SupCategoryData = null;
}
await _dbContext.Categories.AddAsync(new CategoryData
{
Url = category.Url,
Name = category.Name,
SupCategoryData = category.SupCategoryData == null
? null
: _dbContext.Categories.FirstOrDefault(c => c.Name == category.SupCategoryData.Name)
});
await _dbContext.SaveChangesAsync();
}
product.ProductState = parsedProduct.ProductState;
product.Description = parsedProduct.Description;
product.ExpirationDate = parsedProduct.ExpirationDate;
product.JsonData = parsedProduct.JsonData;
product.ExternalId = parsedProduct.ExternalId;
product.JsonDataSchema = parsedProduct.JsonDataSchema;
product.Price = parsedProduct.Price;
product.SyncDate = parsedProduct.SyncDate;
product.Title = parsedProduct.Title;
product.Url = parsedProduct.Url;
product.KeyWords = parsedProduct.KeyWords;
product.ProductAttribute = parsedProduct.ProductAttribute;
product.ProductDeliveryOptions = parsedProduct.ProductDeliveryOptions;
product.ProductPaymentOptions = parsedProduct.ProductPaymentOptions;
product.Presence = parsedProduct.Presence;
if (product.ProductAttribute.Count > 0)
{
foreach (var attribute in product.ProductAttribute)
{
attribute.Product = product;
}
}
if (product.ProductDeliveryOptions.Count > 0)
{
foreach (var deliveryOption in product.ProductDeliveryOptions)
{
deliveryOption.Product = product;
}
}
if (product.ProductPaymentOptions.Count > 0)
{
foreach (var paymentOption in product.ProductPaymentOptions)
{
paymentOption.Product = product;
}
}
if (parsedProduct.Categories != null)
{
foreach (var currentCategory in parsedProduct.Categories)
{
product.Categories.Add(
_dbContext.Categories.FirstOrDefault(c => c.Name == currentCategory.Name));
}
}
await _dbContext.SaveChangesAsync();
return(product);
}
