public async Task <CompanyData> InsertCompanyIntoDb(CompanyData company) { //Number of pages company.CompanyState = CompanyState.Processing; await _dbContext.SaveChangesAsync(); var counter = 1; var page = await _browsingContextService.OpenPageAsync(company.Url.Replace(".html", $";{counter}.html")); //Get all pages for current company while (page.Url != company.Url || counter == 1) { var products = CompanyParsingService.ParseCompanyProducts(company, page).ToArray(); company.Products.AddRange(products); await TaskQueue.QueueBackgroundWorkItemsRangeAsync(products); _logger.LogInformation($"CompanyService new products of {company.Name} company, data-chunk was added"); counter++; Thread.Sleep(5000); page = await _browsingContextService.OpenPageAsync(company.Url.Replace(".html", $";{counter}.html")); if (page != null) { continue; } company.CompanyState = CompanyState.Failed; await _dbContext.SaveChangesAsync(); return(company); } company.CompanyState = CompanyState.Success; company.SyncDate = DateTime.Now; var jsonCompanyDat = new CompanyJson { ExternalId = company.ExternalId, PortalPageUrl = company.Url, SyncDate = company.SyncDate, Name = company.Name }; var generator = new JSchemaGenerator(); var fullCategorySchema = generator.Generate(typeof(CompanyJson)).ToString(); var fullCategoryJson = JsonConvert.SerializeObject(jsonCompanyDat); company.JsonData = fullCategoryJson; company.JsonDataSchema = fullCategorySchema; await _dbContext.SaveChangesAsync(); return(company); }
public async Task <ProductData> InsertProductPageIntoDb(ProductData product) { var productPage = await _browsingContextService.OpenPageAsync(product.Url); if (productPage == null) { throw new NullReferenceException(nameof(productPage)); } if (productPage.StatusCode == HttpStatusCode.TooManyRequests) { throw new TooManyRequestsException(); } var parsedProduct = ProductParsingService.ParseSinglePage(product, productPage); var categories = CategoryParsingService.ParseCategories(product, productPage); var enumerableCategories = categories as CategoryData[] ?? categories.ToArray(); parsedProduct.Categories = enumerableCategories.ToList(); foreach (var category in enumerableCategories.Where(category => _dbContext.Categories.FirstOrDefault(cat => cat.Name == category.Name) == null)) { if (category.SupCategoryData?.SupCategoryData != null) { category.SupCategoryData.SupCategoryData = null; } await _dbContext.Categories.AddAsync(new CategoryData { Url = category.Url, Name = category.Name, SupCategoryData = category.SupCategoryData == null ? null : _dbContext.Categories.FirstOrDefault(c => c.Name == category.SupCategoryData.Name) }); await _dbContext.SaveChangesAsync(); } product.ProductState = parsedProduct.ProductState; product.Description = parsedProduct.Description; product.ExpirationDate = parsedProduct.ExpirationDate; product.JsonData = parsedProduct.JsonData; product.ExternalId = parsedProduct.ExternalId; product.JsonDataSchema = parsedProduct.JsonDataSchema; product.Price = parsedProduct.Price; product.SyncDate = parsedProduct.SyncDate; product.Title = parsedProduct.Title; product.Url = parsedProduct.Url; product.KeyWords = parsedProduct.KeyWords; product.ProductAttribute = parsedProduct.ProductAttribute; product.ProductDeliveryOptions = parsedProduct.ProductDeliveryOptions; product.ProductPaymentOptions = parsedProduct.ProductPaymentOptions; product.Presence = parsedProduct.Presence; if (product.ProductAttribute.Count > 0) { foreach (var attribute in product.ProductAttribute) { attribute.Product = product; } } if (product.ProductDeliveryOptions.Count > 0) { foreach (var deliveryOption in product.ProductDeliveryOptions) { deliveryOption.Product = product; } } if (product.ProductPaymentOptions.Count > 0) { foreach (var paymentOption in product.ProductPaymentOptions) { paymentOption.Product = product; } } if (parsedProduct.Categories != null) { foreach (var currentCategory in parsedProduct.Categories) { product.Categories.Add( _dbContext.Categories.FirstOrDefault(c => c.Name == currentCategory.Name)); } } await _dbContext.SaveChangesAsync(); return(product); }