private static string GetCsvField(string name, ICsvReaderRow row, webModel.CsvImportConfiguration configuration) { var mapping = configuration.MappingItems.First(y => y.EntityColumnName == name); if (mapping == null) { throw new NullReferenceException("mapping"); } var retVal = mapping.CustomValue; if (mapping.CsvColumnName != null) { retVal = row.GetField <string>(mapping.CsvColumnName); } return(retVal); }
public virtual void DoImport(webModel.CsvImportConfiguration configuration, ImportNotification notification) { var csvProducts = new List <coreModel.CatalogProduct>(); var catalog = _catalogService.GetById(configuration.CatalogId); try { using (var reader = new CsvReader(new StreamReader(_blobStorageProvider.OpenReadOnly(configuration.FileUrl)))) { reader.Configuration.Delimiter = configuration.Delimiter; var initialized = false; while (reader.Read()) { if (!initialized) { //Notification notification.Description = "Configuration..."; _notifier.Upsert(notification); //set import configuration based on mapping info var productMap = new ProductMap(reader.FieldHeaders, configuration, catalog); reader.Configuration.RegisterClassMap(productMap); initialized = true; //Notification notification.Description = "Reading products from csv..."; _notifier.Upsert(notification); } try { var csvProduct = reader.GetRecord <coreModel.CatalogProduct>(); csvProducts.Add(csvProduct); } catch (Exception ex) { var error = ex.Message; if (ex.Data.Contains("CsvHelper")) { error += ex.Data["CsvHelper"]; } notification.ErrorCount++; notification.Errors.Add(error); } } } SaveCategoryTree(catalog, csvProducts, notification); SaveProducts(catalog, csvProducts, notification); } catch (Exception ex) { notification.Description = "Export error"; notification.ErrorCount++; notification.Errors.Add(ex.ToString()); } finally { notification.Finished = DateTime.UtcNow; notification.Description = "Import finished" + (notification.Errors.Any() ? " with errors" : " successfully"); _notifier.Upsert(notification); } }
public IHttpActionResult GetMappingConfiguration([FromUri]string fileUrl, [FromUri]string delimiter = ";") { var retVal = new CsvImportConfiguration { Delimiter = delimiter, FileUrl = fileUrl }; var mappingItems = new List<CsvImportMappingItem>(); mappingItems.AddRange(ReflectionUtility.GetPropertyNames<coreModel.CatalogProduct>(x => x.Name, x => x.Category).Select(x => new CsvImportMappingItem { EntityColumnName = x, IsRequired = true })); mappingItems.AddRange(new string[] {"Sku", "ParentSku", "Review", "PrimaryImage", "AltImage", "SeoUrl", "SeoDescription", "SeoTitle", "PriceId", "Price", "SalePrice", "Currency", "AllowBackorder", "Quantity", "FulfilmentCenterId" } .Select(x => new CsvImportMappingItem { EntityColumnName = x, IsRequired = false })); mappingItems.AddRange(ReflectionUtility.GetPropertyNames<coreModel.CatalogProduct>(x => x.Id, x => x.MainProductId, x => x.CategoryId, x => x.IsActive, x => x.IsBuyable, x => x.TrackInventory, x => x.ManufacturerPartNumber, x => x.Gtin, x => x.MeasureUnit, x => x.WeightUnit, x => x.Weight, x => x.Height, x => x.Length, x => x.Width, x => x.TaxType, x => x.ProductType, x => x.ShippingType, x => x.Vendor, x => x.DownloadType, x => x.DownloadExpiration, x => x.HasUserAgreement).Select(x => new CsvImportMappingItem { EntityColumnName = x, IsRequired = false })); retVal.MappingItems = mappingItems.ToArray(); //Read csv headers and try to auto map fields by name using (var reader = new CsvReader(new StreamReader(_blobStorageProvider.OpenReadOnly(fileUrl)))) { reader.Configuration.Delimiter = delimiter; while (reader.Read()) { var csvColumns = reader.FieldHeaders; retVal.CsvColumns = csvColumns; //default columns mapping if (csvColumns.Any()) { foreach (var mappingItem in retVal.MappingItems) { var entityColumnName = mappingItem.EntityColumnName; var betterMatchCsvColumn = csvColumns.Select(x => new { csvColumn = x, distance = x.ComputeLevenshteinDistance(entityColumnName) }) .Where(x => x.distance < 2) .OrderBy(x => x.distance) .Select(x => x.csvColumn) .FirstOrDefault(); if (betterMatchCsvColumn != null) { mappingItem.CsvColumnName = betterMatchCsvColumn; mappingItem.CustomValue = null; } } } } } //All not mapped properties may be a product property retVal.PropertyCsvColumns = retVal.CsvColumns.Except(retVal.MappingItems.Where(x => x.CsvColumnName != null).Select(x => x.CsvColumnName)).ToArray(); //Generate ETag for identifying csv format retVal.ETag = string.Join(";", retVal.CsvColumns).GetMD5Hash(); return Ok(retVal); }
public ProductMap(string[] allColumns, webModel.CsvImportConfiguration importConfiguration, coreModel.Catalog catalog) { var defaultLanguge = catalog.DefaultLanguage != null ? catalog.DefaultLanguage.LanguageCode : "EN-US"; //Dynamical map scalar product fields use by manual mapping information foreach (var mappingConfigurationItem in importConfiguration.MappingItems.Where(x => x.CsvColumnName != null || x.CustomValue != null)) { var propertyInfo = typeof(coreModel.CatalogProduct).GetProperty(mappingConfigurationItem.EntityColumnName); if (propertyInfo != null) { var newMap = new CsvPropertyMap(propertyInfo); newMap.TypeConverterOption(CultureInfo.InvariantCulture); newMap.TypeConverterOption(NumberStyles.Any); newMap.TypeConverterOption(true, "yes", "false"); //Map fields if mapping specified if (mappingConfigurationItem.CsvColumnName != null) { newMap.Name(mappingConfigurationItem.CsvColumnName); } //And default values if it specified if (mappingConfigurationItem.CustomValue != null) { newMap.Default(mappingConfigurationItem.CustomValue); } PropertyMaps.Add(newMap); } } //Map Sku -> Code Map(x => x.Code).ConvertUsing(x => { return(GetCsvField("Sku", x, importConfiguration)); }); //Map ParentSku -> main product Map(x => x.MainProductId).ConvertUsing(x => { var parentSku = GetCsvField("ParentSku", x, importConfiguration); return(!String.IsNullOrEmpty(parentSku) ? parentSku : null); }); //Map assets (images) Map(x => x.Assets).ConvertUsing(x => { var retVal = new List <coreModel.ItemAsset>(); var primaryImageUrl = GetCsvField("PrimaryImage", x, importConfiguration); var altImageUrl = GetCsvField("AltImage", x, importConfiguration); if (!String.IsNullOrEmpty(primaryImageUrl)) { retVal.Add(new coreModel.ItemAsset { Type = coreModel.ItemAssetType.Image, Group = "primaryimage", Url = primaryImageUrl }); } if (!String.IsNullOrEmpty(altImageUrl)) { retVal.Add(new coreModel.ItemAsset { Type = coreModel.ItemAssetType.Image, Group = "image", Url = altImageUrl }); } return(retVal); }); //Map category Map(x => x.Category).ConvertUsing(x => new coreModel.Category { Path = GetCsvField("Category", x, importConfiguration) }); //Map Reviews Map(x => x.Reviews).ConvertUsing(x => { var reviews = new List <coreModel.EditorialReview>(); var content = GetCsvField("Review", x, importConfiguration); if (!String.IsNullOrEmpty(content)) { reviews.Add(new coreModel.EditorialReview { Content = content, LanguageCode = defaultLanguge }); } return(reviews); }); //Map Seo Map(x => x.SeoInfos).ConvertUsing(x => { var seoInfos = new List <SeoInfo>(); var seoUrl = GetCsvField("SeoUrl", x, importConfiguration); var seoDescription = GetCsvField("SeoDescription", x, importConfiguration); var seoTitle = GetCsvField("SeoTitle", x, importConfiguration); if (!String.IsNullOrEmpty(seoUrl) || !String.IsNullOrEmpty(seoTitle) || !String.IsNullOrEmpty(seoDescription)) { seoUrl = new string[] { seoUrl, seoDescription, seoTitle }.Where(y => !String.IsNullOrEmpty(y)).FirstOrDefault(); seoUrl = seoUrl.Substring(0, Math.Min(seoUrl.Length, 240)); seoInfos.Add(new SeoInfo { LanguageCode = defaultLanguge, SemanticUrl = seoUrl.GenerateSlug(), MetaDescription = seoDescription, PageTitle = seoTitle }); } return(seoInfos); }); //Map Prices Map(x => x.Prices).ConvertUsing(x => { var prices = new List <Price>(); var priceId = GetCsvField("PriceId", x, importConfiguration); var listPrice = GetCsvField("Price", x, importConfiguration); var salePrice = GetCsvField("SalePrice", x, importConfiguration); var currency = GetCsvField("Currency", x, importConfiguration); if (!String.IsNullOrEmpty(listPrice) && !String.IsNullOrEmpty(currency)) { prices.Add(new Price { Id = priceId, List = Convert.ToDecimal(listPrice, CultureInfo.InvariantCulture), Sale = salePrice != null ? (decimal?)Convert.ToDecimal(listPrice, CultureInfo.InvariantCulture) : null, Currency = EnumUtility.SafeParse <CurrencyCodes>(currency, CurrencyCodes.USD) }); } return(prices); }); //Map inventories Map(x => x.Inventories).ConvertUsing(x => { var inventories = new List <InventoryInfo>(); var quantity = GetCsvField("Quantity", x, importConfiguration); var allowBackorder = GetCsvField("AllowBackorder", x, importConfiguration); var fulfilmentCenterId = GetCsvField("FulfilmentCenterId", x, importConfiguration); if (!String.IsNullOrEmpty(quantity)) { inventories.Add(new InventoryInfo { FulfillmentCenterId = fulfilmentCenterId, AllowBackorder = allowBackorder.TryParse(false), InStockQuantity = (long)quantity.TryParse(0.0m) }); } return(inventories); }); //Map properties if (importConfiguration.PropertyCsvColumns != null && importConfiguration.PropertyCsvColumns.Any()) { Map(x => x.PropertyValues).ConvertUsing(x => importConfiguration.PropertyCsvColumns.Select(column => new coreModel.PropertyValue { PropertyName = column, Value = x.GetField <string>(column) }).ToList()); } }
public IHttpActionResult DoImport(CsvImportConfiguration importConfiguration) { var notification = new ImportNotification(CurrentPrincipal.GetCurrentUserName()) { Title = "Import catalog from CSV", Description = "starting import...." }; _notifier.Upsert(notification); var importJob = new CsvCatalogImportJob(); BackgroundJob.Enqueue(() => importJob.DoImport(importConfiguration, notification)); return Ok(notification); }