// Only public methods can be invoked in the background. (Hangfire) public void BackgroundImport(CsvImportInfo importInfo, ImportNotification notifyEvent) { Action <ExportImportProgressInfo> progressCallback = x => { notifyEvent.InjectFrom(x); _notifier.Upsert(notifyEvent); }; using (var stream = _blobStorageProvider.OpenRead(importInfo.FileUrl)) { try { _csvImporter.DoImport(stream, importInfo, progressCallback); } catch (Exception ex) { notifyEvent.Description = "Export error"; notifyEvent.ErrorCount++; notifyEvent.Errors.Add(ex.ToString()); } finally { notifyEvent.Finished = DateTime.UtcNow; notifyEvent.Description = "Import finished" + (notifyEvent.Errors.Any() ? " with errors" : " successfully"); _notifier.Upsert(notifyEvent); } } }
// Support methods private List <CsvProduct> ReadCsvFile(string path, CsvImportInfo importInfo) { var csvProducts = new List <CsvProduct>(); #pragma warning disable S3966 // Objects should not be disposed more than once - no problem in this case using (var fs = File.Open(path, FileMode.Open)) #pragma warning restore S3966 // Objects should not be disposed more than once { using (var reader = new CsvReader(new StreamReader(fs))) { reader.Configuration.Delimiter = importInfo.Configuration.Delimiter; reader.Configuration.RegisterClassMap(new CsvProductMap(importInfo.Configuration)); reader.Configuration.MissingFieldFound = (strings, i, arg3) => { //do nothing }; reader.Configuration.TrimOptions = TrimOptions.Trim; reader.Configuration.HeaderValidated = null; while (reader.Read()) { var csvProduct = reader.GetRecord <CsvProduct>(); csvProducts.Add(csvProduct); } } } return(csvProducts); }
private List <CsvProduct> ReadCsvFile(string path, CsvImportInfo importInfo) { var csvProducts = new List <CsvProduct>(); using (FileStream fs = File.Open(path, FileMode.Open)) { using (var reader = new CsvReader(new StreamReader(fs))) { reader.Configuration.Delimiter = importInfo.Configuration.Delimiter; reader.Configuration.RegisterClassMap(new CsvProductMap(importInfo.Configuration)); reader.Configuration.MissingFieldFound = (strings, i, arg3) => { //do nothing }; reader.Configuration.TrimOptions = TrimOptions.Trim; reader.Configuration.HeaderValidated = null; while (reader.Read()) { var csvProduct = reader.GetRecord <CsvProduct>(); csvProducts.Add(csvProduct); } } } return(csvProducts); }
public void CsvProductMapTest_CsvHasPropertyValues_PropertyValuesMapped() { var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.CsvColumns = new[] { "Sku" }; importInfo.Configuration.PropertyCsvColumns = new[] { "ProductProperty", "ProductProperty_Multivalue" }; importInfo.Configuration.Delimiter = ","; string path = @"../../data/product-propertyvalues.csv"; var csvProducts = ReadCsvFile(path, importInfo); Action <PropertyValue>[] inspectorsFirstProduct = { x => Assert.True((string)x.Value == "Product-1-propertyvalue-test" && x.PropertyName == "ProductProperty"), x => Assert.True((string)x.Value == "Product-1-multivalue-1, Product-1-multivalue-2" && x.PropertyName == "ProductProperty_Multivalue") }; Action <PropertyValue>[] inspectorsSecond = { x => Assert.True((string)x.Value == "Product-2-propertyvalue-test" && x.PropertyName == "ProductProperty"), x => Assert.True((string)x.Value == "Product-2-multivalue-1, Product-2-multivalue-1, Product-2-multivalue-3" && x.PropertyName == "ProductProperty_Multivalue") }; Assert.Collection(csvProducts.FirstOrDefault().PropertyValues, inspectorsFirstProduct); Assert.Collection(csvProducts.LastOrDefault().PropertyValues, inspectorsSecond); }
public void CsvProductMapTest_CsvHasProductProperties_PropertiesMapped() { var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.Delimiter = ","; string path = @"../../data/product-productproperties.csv"; var csvProducts = ReadCsvFile(path, importInfo); var product = csvProducts.FirstOrDefault(); Assert.Equal("429408", product.Id); Assert.Equal("CBLK21113", product.Sku); Assert.Equal("cblk21113-product-1", product.Name); Assert.Equal("catId_1", product.CategoryId); Assert.Equal("GTIN_Value", product.Gtin); Assert.Equal("mainprod_123", product.MainProductId); Assert.Equal("Vendor_value", product.Vendor); Assert.Equal("ProductType_value", product.ProductType); Assert.Equal("ShippingType_value", product.ShippingType); Assert.Equal("DownloadType_value", product.DownloadType); Assert.True(product.HasUserAgreement); Assert.True(product.IsBuyable); Assert.True(product.TrackInventory); }
public void CsvProductMapTest_CsvHasProductProperties_PropertiesMapped() { var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.Delimiter = ","; string path = GetDataFilePath("product-productproperties.csv"); var csvProducts = ReadCsvFile(path, importInfo); var product = csvProducts.FirstOrDefault(); Assert.Equal("429408", product.Id); Assert.Equal("CBLK21113", product.Sku); Assert.Equal("cblk21113-product-1", product.Name); Assert.Equal("catId_1", product.CategoryId); Assert.Equal("GTIN_Value", product.Gtin); Assert.Equal("mainprod_123", product.MainProductId); Assert.Equal("Vendor_value", product.Vendor); Assert.Equal("ProductType_value", product.ProductType); Assert.Equal("ShippingType_value", product.ShippingType); Assert.Equal("DownloadType_value", product.DownloadType); Assert.Equal("OuterId", product.OuterId); Assert.Equal(1, product.Priority); Assert.Equal(10, product.MaxQuantity); Assert.Equal(5, product.MinQuantity); Assert.Equal("PackageType", product.PackageType); Assert.Equal("FulfillmentCenterId", product.FulfillmentCenterId); Assert.Equal(1, product.MaxNumberOfDownload); Assert.True(product.HasUserAgreement); Assert.True(product.IsBuyable); Assert.True(product.TrackInventory); }
public async Task DoImportAsync(Stream inputStream, CsvImportInfo importInfo, Action <ExportImportProgressInfo> progressCallback) { var csvProducts = new List <CsvProduct>(); var progressInfo = new ExportImportProgressInfo { Description = "Reading products from csv..." }; progressCallback(progressInfo); var encoding = DetectEncoding(inputStream); using (var reader = new CsvReader(new StreamReader(inputStream, encoding))) { reader.Configuration.Delimiter = importInfo.Configuration.Delimiter; reader.Configuration.RegisterClassMap(new CsvProductMap(importInfo.Configuration)); reader.Configuration.MissingFieldFound = (strings, i, arg3) => { //do nothing }; reader.Configuration.TrimOptions = TrimOptions.Trim; while (reader.Read()) { try { var csvProduct = reader.GetRecord <CsvProduct>(); ReplaceEmptyStringsWithNull(csvProduct); csvProducts.Add(csvProduct); } catch (TypeConverterException ex) { progressInfo.Errors.Add($"Column: {ex.MemberMapData.Member.Name}, {ex.Message}"); progressCallback(progressInfo); } catch (Exception ex) { var error = ex.Message; if (ex.Data.Contains("CsvHelper")) { error += ex.Data["CsvHelper"]; } progressInfo.Errors.Add(error); progressCallback(progressInfo); } } } await DoImport(csvProducts, importInfo, progressInfo, progressCallback); }
public void CsvProductMapTest_CsvHasMultipleLines_LineNumberMapTest() { var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.Delimiter = ","; string path = @"../../data/product-productproperties-twoproducts.csv"; var csvProducts = ReadCsvFile(path, importInfo); Assert.Equal(2, csvProducts[0].LineNumber); Assert.Equal(3, csvProducts[1].LineNumber); }
public IHttpActionResult DoImport(CsvImportInfo importInfo) { var notification = new ImportNotification(CurrentPrincipal.GetCurrentUserName()) { Title = "Import catalog from CSV", Description = "starting import...." }; _notifier.Upsert(notification); BackgroundJob.Enqueue(() => BackgroundImport(importInfo, notification)); return(Ok(notification)); }
public void CsvProductMapTest_CsvHasCategorypath_CategoryPathMapped() { var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.Delimiter = ","; string path = @"../../data/product-productproperties-categoryPath.csv"; var csvProducts = ReadCsvFile(path, importInfo); var product = csvProducts.FirstOrDefault(); Assert.Equal("TestCategory1", product.CategoryPath); Assert.Equal("TestCategory1", product.Category.Path); }
public IHttpActionResult DoImport(CsvImportInfo importInfo) { CheckCurrentUserHasPermissionForObjects(CatalogPredefinedPermissions.Import, importInfo); var notification = new ImportNotification(_userNameResolver.GetCurrentUserName()) { Title = "Import catalog from CSV", Description = "starting import...." }; _notifier.Upsert(notification); BackgroundJob.Enqueue(() => BackgroundImport(importInfo, notification)); return(Ok(notification)); }
public async Task DoImport(List <CsvProduct> csvProducts, CsvImportInfo importInfo, ExportImportProgressInfo progressInfo, Action <ExportImportProgressInfo> progressCallback) { var catalog = (await _catalogService.GetByIdsAsync(new[] { importInfo.CatalogId })).FirstOrDefault(); if (catalog == null) { throw new InvalidOperationException($"Catalog with id \"{importInfo.CatalogId}\" does not exist."); } _stores.AddRange((await _storeSearchService.SearchStoresAsync(new StoreSearchCriteria { Take = int.MaxValue })).Stores); var contunie = ImportAllowed(csvProducts, progressInfo, progressCallback); if (!contunie) { return; } csvProducts = MergeCsvProducts(csvProducts, catalog); await MergeFromAlreadyExistProducts(csvProducts, catalog); await SaveCategoryTree(catalog, csvProducts, progressInfo, progressCallback); await LoadProductDependencies(csvProducts, catalog, importInfo); await ResolvePropertyDictionaryItems(csvProducts, progressInfo, progressCallback); //take parentless prodcuts and save them first progressInfo.TotalCount = csvProducts.Count; var mainProcuts = csvProducts.Where(x => x.MainProduct == null).ToList(); await SaveProducts(mainProcuts, progressInfo, progressCallback); //prepare and save variations (needed to be able to save variation with SKU as MainProductId) var variations = csvProducts.Except(mainProcuts).ToList(); foreach (var variation in variations.Where(x => x.MainProductId == null)) { variation.MainProductId = variation.MainProduct.Id; } await SaveProducts(variations, progressInfo, progressCallback); }
public void DoImport(Stream inputStream, CsvImportInfo importInfo, Action <ExportImportProgressInfo> progressCallback) { var csvProducts = new List <CsvProduct>(); var progressInfo = new ExportImportProgressInfo { Description = "Reading products from csv..." }; progressCallback(progressInfo); var encoding = DetectEncoding(inputStream); using (var reader = new CsvReader(new StreamReader(inputStream, encoding))) { reader.Configuration.Delimiter = importInfo.Configuration.Delimiter; reader.Configuration.RegisterClassMap(new CsvProductMap(importInfo.Configuration)); reader.Configuration.MissingFieldFound = (strings, i, arg3) => { //do nothing }; reader.Configuration.TrimOptions = TrimOptions.Trim; while (reader.Read()) { try { var csvProduct = reader.GetRecord <CsvProduct>(); csvProduct.Id = string.IsNullOrEmpty(csvProduct.Id) ? null : csvProduct.Id; csvProducts.Add(csvProduct); } catch (Exception ex) { var error = ex.Message; if (ex.Data.Contains("CsvHelper")) { error += ex.Data["CsvHelper"]; } progressInfo.Errors.Add(error); progressCallback(progressInfo); } } } DoImport(csvProducts, importInfo, progressInfo, progressCallback); }
public void CsvProductMapTest_CsvHasReview_ReviewMapped() { var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.Delimiter = ","; string path = @"../../data/product-productproperties-review.csv"; var csvProducts = ReadCsvFile(path, importInfo); var product = csvProducts.FirstOrDefault(); Assert.Equal("Review_Content", product.Review); Assert.Equal("Review_Content", product.EditorialReview.Content); Assert.Equal("ReviewType_Value", product.ReviewType); Assert.Equal("ReviewType_Value", product.EditorialReview.ReviewType); }
public void CsvProductMapTest_CsvHasBooleanValues_BooleanFieldsMapped() { var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.Delimiter = ","; string path = @"../../data/product-productproperties-boolean.csv"; var csvProducts = ReadCsvFile(path, importInfo); Assert.False(csvProducts[0].HasUserAgreement); Assert.False(csvProducts[0].IsBuyable); Assert.False(csvProducts[0].TrackInventory); Assert.True(csvProducts[1].HasUserAgreement); Assert.True(csvProducts[1].IsBuyable); Assert.True(csvProducts[1].TrackInventory); }
public async Task <ActionResult <ImportNotification> > DoImport([FromBody] CsvImportInfo importInfo) { var hasPermissions = true; if (!importInfo.CatalogId.IsNullOrEmpty()) { var catalogs = await _catalogService.GetByIdsAsync(new[] { importInfo.CatalogId }, CategoryResponseGroup.Info.ToString()); if (!catalogs.IsNullOrEmpty()) { hasPermissions = await CheckCatalogPermission(catalogs.First(), CatalogModuleConstants.Security.Permissions.Update); } } if (!hasPermissions) { return(Unauthorized()); } var criteria = AbstractTypeFactory <CatalogSearchCriteria> .TryCreateInstance(); criteria.CatalogIds = new[] { importInfo.CatalogId }; var authorizationResult = await _authorizationService.AuthorizeAsync(User, criteria, new CatalogAuthorizationRequirement(CatalogModuleConstants.Security.Permissions.Update)); if (!authorizationResult.Succeeded) { return(Unauthorized()); } var notification = new ImportNotification(_userNameResolver.GetCurrentUserName()) { Title = "Import catalog from CSV", Description = "starting import...." }; await _notifier.SendAsync(notification); BackgroundJob.Enqueue(() => BackgroundImport(importInfo, notification)); return(Ok(notification)); }
public void CsvProductMapTest_CsvHasSeoInfo_SeoInfoMapped() { var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.Delimiter = ","; string path = @"../../data/product-productproperties-seoInfo.csv"; var csvProducts = ReadCsvFile(path, importInfo); var product = csvProducts.FirstOrDefault(x => x.Code == "cblk21113-product-1"); Assert.Equal("seo-slug-url", product.SeoUrl); Assert.Equal("seo-slug-url", product.SeoInfo.SemanticUrl); Assert.Equal("Seo_Title_Value", product.SeoTitle); Assert.Equal("Seo_Title_Value", product.SeoInfo.PageTitle); Assert.Equal("Seo_Descr_Value", product.SeoDescription); Assert.Equal("Seo_Descr_Value", product.SeoInfo.MetaDescription); Assert.Equal("Seo_Language_Value", product.SeoInfo.LanguageCode); Assert.True(csvProducts.Count == 2); }
public void CsvProductMapTest_CsvHasPriceAndQuantity_PriceAndQuantityMapped() { var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.Delimiter = ","; string path = @"../../data/product-productproperties-priceQuantity.csv"; var csvProducts = ReadCsvFile(path, importInfo); var product = csvProducts.FirstOrDefault(); Assert.Equal("123.4", product.ListPrice); Assert.Equal(123.4m, product.Price.List); Assert.Equal("456.7", product.SalePrice); Assert.Equal(456.7m, product.Price.Sale); Assert.Equal("EUR", product.Currency); Assert.Equal("EUR", product.Price.Currency); Assert.Equal("5", product.PriceMinQuantity); Assert.Equal(5, product.Price.MinQuantity); }
public void CsvProductMapTest_MappingHasDefaultBoolValue_DefaultBoolValuesMapped() { var defaultValue = true; var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.Delimiter = ","; var categoryPathMapping = importInfo.Configuration.PropertyMaps.FirstOrDefault(x => x.EntityColumnName == "IsBuyable"); categoryPathMapping.CsvColumnName = null; categoryPathMapping.CustomValue = defaultValue.ToString(); string path = @"../../data/product-productproperties.csv"; var csvProducts = ReadCsvFile(path, importInfo); var product = csvProducts.FirstOrDefault(); Assert.True(product.IsBuyable); }
public void CsvProductMapTest_MappingHasDefaultCategoryPath_DefaultCategoryPathMapped() { var defaultValue = "Custom_category_path_value"; var importInfo = new CsvImportInfo { Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration() }; importInfo.Configuration.Delimiter = ","; var categoryPathMapping = importInfo.Configuration.PropertyMaps.FirstOrDefault(x => x.EntityColumnName == "CategoryPath"); categoryPathMapping.CsvColumnName = null; categoryPathMapping.CustomValue = defaultValue; string path = @"../../data/product-productproperties-noCategoryPath.csv"; var csvProducts = ReadCsvFile(path, importInfo); var product = csvProducts.FirstOrDefault(); Assert.Equal(defaultValue, product.CategoryPath); Assert.Equal(defaultValue, product.Category.Path); }
public void DoImport(List <CsvProduct> csvProducts, CsvImportInfo importInfo, ExportImportProgressInfo progressInfo, Action <ExportImportProgressInfo> progressCallback) { var catalog = _catalogService.GetById(importInfo.CatalogId); _stores.AddRange(_storeService.SearchStores(new Domain.Store.Model.SearchCriteria { Take = int.MaxValue }).Stores); var contunie = ImportAllowed(csvProducts, progressInfo, progressCallback); if (!contunie) { return; } csvProducts = MergeCsvProducts(csvProducts, catalog); MergeFromAlreadyExistProducts(csvProducts, catalog); SaveCategoryTree(catalog, csvProducts, progressInfo, progressCallback); LoadProductDependencies(csvProducts, catalog, progressInfo, progressCallback, importInfo); ResolvePropertyDictionaryItems(csvProducts, progressInfo, progressCallback); //take parentless prodcuts and save them first progressInfo.TotalCount = csvProducts.Count; var mainProcuts = csvProducts.Where(x => x.MainProduct == null).ToList(); SaveProducts(mainProcuts, progressInfo, progressCallback); //prepare and save variations (needed to be able to save variation with SKU as MainProductId) var variations = csvProducts.Except(mainProcuts).ToList(); variations.Where(x => x.MainProductId == null).ForEach(x => x.MainProductId = x.MainProduct.Id); SaveProducts(variations, progressInfo, progressCallback); }
private async Task LoadProductDependencies(IEnumerable <CsvProduct> csvProducts, Catalog catalog, CsvImportInfo importInfo) { var allCategoriesIds = csvProducts.Select(x => x.CategoryId).Distinct().ToArray(); var categoriesMap = (await _categoryService.GetByIdsAsync(allCategoriesIds, CategoryResponseGroup.Full.ToString())).ToDictionary(x => x.Id); foreach (var csvProduct in csvProducts) { csvProduct.Catalog = catalog; csvProduct.CatalogId = catalog.Id; if (csvProduct.CategoryId != null) { csvProduct.Category = categoriesMap[csvProduct.CategoryId]; } //Try to set parent relations //By id or code reference var parentProduct = csvProducts.FirstOrDefault(x => !string.IsNullOrEmpty(csvProduct.MainProductId) && (x.Id.EqualsInvariant(csvProduct.MainProductId) || x.Code.EqualsInvariant(csvProduct.MainProductId))); csvProduct.MainProduct = parentProduct; csvProduct.MainProductId = parentProduct != null ? parentProduct.Id : null; if (string.IsNullOrEmpty(csvProduct.Code)) { csvProduct.Code = _skuGenerator.GenerateSku(csvProduct); } //Properties inheritance var inheritedProperties = (csvProduct.Category != null ? csvProduct.Category.Properties : csvProduct.Catalog.Properties).OrderBy(x => x.Name).ToList(); foreach (var property in csvProduct.Properties.ToArray()) { //Try to find property for product var inheritedProperty = inheritedProperties.FirstOrDefault(x => x.Name.EqualsInvariant(property.Name)); if (inheritedProperty != null) { property.ValueType = inheritedProperty.ValueType; property.Id = inheritedProperty.Id; property.Dictionary = inheritedProperty.Dictionary; property.Multivalue = inheritedProperty.Multivalue; foreach (var propertyValue in property.Values) { propertyValue.ValueType = inheritedProperty.ValueType; propertyValue.PropertyId = inheritedProperty.Id; } //Try to split the one value to multiple values for Multivalue properties if (inheritedProperty.Multivalue) { var parsedValues = new List <PropertyValue>(); foreach (var propertyValue in property.Values) { parsedValues.AddRange(ParseValuesFromMultivalueString(propertyValue, importInfo.Configuration.Delimiter)); } property.Values = parsedValues; } } } } }
private void LoadProductDependencies(IEnumerable <CsvProduct> csvProducts, Catalog catalog, ExportImportProgressInfo progressInfo, Action <ExportImportProgressInfo> progressCallback, CsvImportInfo importInfo) { var modifiedProperties = new List <Property>(); var allCategoriesIds = csvProducts.Select(x => x.CategoryId).Distinct().ToArray(); var categoriesMap = _categoryService.GetByIds(allCategoriesIds, CategoryResponseGroup.Full).ToDictionary(x => x.Id); foreach (var csvProduct in csvProducts) { csvProduct.Catalog = catalog; csvProduct.CatalogId = catalog.Id; if (csvProduct.CategoryId != null) { csvProduct.Category = categoriesMap[csvProduct.CategoryId]; } //Try to set parent relations //By id or code reference var parentProduct = csvProducts.FirstOrDefault(x => !string.IsNullOrEmpty(csvProduct.MainProductId) && (x.Id.EqualsInvariant(csvProduct.MainProductId) || x.Code.EqualsInvariant(csvProduct.MainProductId))); csvProduct.MainProduct = parentProduct; csvProduct.MainProductId = parentProduct != null ? parentProduct.Id : null; if (string.IsNullOrEmpty(csvProduct.Code)) { csvProduct.Code = _skuGenerator.GenerateSku(csvProduct); } //Properties inheritance csvProduct.Properties = (csvProduct.Category != null ? csvProduct.Category.Properties : csvProduct.Catalog.Properties).OrderBy(x => x.Name).ToList(); foreach (var propertyValue in csvProduct.PropertyValues.ToArray()) { //Try to find property for product propertyValue.Property = csvProduct.Properties.FirstOrDefault(x => x.Name.EqualsInvariant(propertyValue.PropertyName)); if (propertyValue.Property != null) { propertyValue.ValueType = propertyValue.Property.ValueType; propertyValue.PropertyId = propertyValue.Property.Id; //Try to split the one value to multiple values for Multivalue properties if (propertyValue.Property.Multivalue) { var multivalue = propertyValue.Value.ToString(); var chars = new[] { ",", importInfo.Configuration.Delimiter }; var values = multivalue.Split(chars, StringSplitOptions.RemoveEmptyEntries) .Select(x => x.Trim()) .Where(x => !string.IsNullOrEmpty(x)) .Distinct().ToArray(); propertyValue.Value = values.FirstOrDefault(); foreach (var value in values.Skip(1)) { var newPropValue = propertyValue.Clone() as PropertyValue; newPropValue.Value = value; csvProduct.PropertyValues.Add(newPropValue); } } } } } }