private static string GetCsvField(string name, ICsvReaderRow row, webModel.CsvImportConfiguration configuration)
{
var mapping = configuration.MappingItems.First(y => y.EntityColumnName == name);
if (mapping == null)
{
throw new NullReferenceException("mapping");
}
var retVal = mapping.CustomValue;
if (mapping.CsvColumnName != null)
{
retVal = row.GetField <string>(mapping.CsvColumnName);
}
return(retVal);
}
public virtual void DoImport(webModel.CsvImportConfiguration configuration, ImportNotification notification)
{
var csvProducts = new List <coreModel.CatalogProduct>();
var catalog = _catalogService.GetById(configuration.CatalogId);
try
{
using (var reader = new CsvReader(new StreamReader(_blobStorageProvider.OpenReadOnly(configuration.FileUrl))))
{
reader.Configuration.Delimiter = configuration.Delimiter;
var initialized = false;
while (reader.Read())
{
if (!initialized)
{
//Notification
notification.Description = "Configuration...";
_notifier.Upsert(notification);
//set import configuration based on mapping info
var productMap = new ProductMap(reader.FieldHeaders, configuration, catalog);
reader.Configuration.RegisterClassMap(productMap);
initialized = true;
//Notification
notification.Description = "Reading products from csv...";
_notifier.Upsert(notification);
}
try
{
var csvProduct = reader.GetRecord <coreModel.CatalogProduct>();
csvProducts.Add(csvProduct);
}
catch (Exception ex)
{
var error = ex.Message;
if (ex.Data.Contains("CsvHelper"))
{
error += ex.Data["CsvHelper"];
}
notification.ErrorCount++;
notification.Errors.Add(error);
}
}
}
SaveCategoryTree(catalog, csvProducts, notification);
SaveProducts(catalog, csvProducts, notification);
}
catch (Exception ex)
{
notification.Description = "Export error";
notification.ErrorCount++;
notification.Errors.Add(ex.ToString());
}
finally
{
notification.Finished = DateTime.UtcNow;
notification.Description = "Import finished" + (notification.Errors.Any() ? " with errors" : " successfully");
_notifier.Upsert(notification);
}
}
public IHttpActionResult GetMappingConfiguration([FromUri]string fileUrl, [FromUri]string delimiter = ";")
{
var retVal = new CsvImportConfiguration
{
Delimiter = delimiter,
FileUrl = fileUrl
};
var mappingItems = new List<CsvImportMappingItem>();
mappingItems.AddRange(ReflectionUtility.GetPropertyNames<coreModel.CatalogProduct>(x => x.Name, x => x.Category).Select(x => new CsvImportMappingItem { EntityColumnName = x, IsRequired = true }));
mappingItems.AddRange(new string[] {"Sku", "ParentSku", "Review", "PrimaryImage", "AltImage", "SeoUrl", "SeoDescription", "SeoTitle",
"PriceId", "Price", "SalePrice", "Currency", "AllowBackorder", "Quantity", "FulfilmentCenterId" }
.Select(x => new CsvImportMappingItem { EntityColumnName = x, IsRequired = false }));
mappingItems.AddRange(ReflectionUtility.GetPropertyNames<coreModel.CatalogProduct>(x => x.Id, x => x.MainProductId, x => x.CategoryId, x => x.IsActive, x => x.IsBuyable, x => x.TrackInventory,
x => x.ManufacturerPartNumber, x => x.Gtin, x => x.MeasureUnit, x => x.WeightUnit, x => x.Weight,
x => x.Height, x => x.Length, x => x.Width, x => x.TaxType, x => x.ProductType, x => x.ShippingType,
x => x.Vendor, x => x.DownloadType, x => x.DownloadExpiration, x => x.HasUserAgreement).Select(x => new CsvImportMappingItem { EntityColumnName = x, IsRequired = false }));
retVal.MappingItems = mappingItems.ToArray();
//Read csv headers and try to auto map fields by name
using (var reader = new CsvReader(new StreamReader(_blobStorageProvider.OpenReadOnly(fileUrl))))
{
reader.Configuration.Delimiter = delimiter;
while (reader.Read())
{
var csvColumns = reader.FieldHeaders;
retVal.CsvColumns = csvColumns;
//default columns mapping
if (csvColumns.Any())
{
foreach (var mappingItem in retVal.MappingItems)
{
var entityColumnName = mappingItem.EntityColumnName;
var betterMatchCsvColumn = csvColumns.Select(x => new { csvColumn = x, distance = x.ComputeLevenshteinDistance(entityColumnName) })
.Where(x => x.distance < 2)
.OrderBy(x => x.distance)
.Select(x => x.csvColumn)
.FirstOrDefault();
if (betterMatchCsvColumn != null)
{
mappingItem.CsvColumnName = betterMatchCsvColumn;
mappingItem.CustomValue = null;
}
}
}
}
}
//All not mapped properties may be a product property
retVal.PropertyCsvColumns = retVal.CsvColumns.Except(retVal.MappingItems.Where(x => x.CsvColumnName != null).Select(x => x.CsvColumnName)).ToArray();
//Generate ETag for identifying csv format
retVal.ETag = string.Join(";", retVal.CsvColumns).GetMD5Hash();
return Ok(retVal);
}
public ProductMap(string[] allColumns, webModel.CsvImportConfiguration importConfiguration, coreModel.Catalog catalog)
{
var defaultLanguge = catalog.DefaultLanguage != null ? catalog.DefaultLanguage.LanguageCode : "EN-US";
//Dynamical map scalar product fields use by manual mapping information
foreach (var mappingConfigurationItem in importConfiguration.MappingItems.Where(x => x.CsvColumnName != null || x.CustomValue != null))
{
var propertyInfo = typeof(coreModel.CatalogProduct).GetProperty(mappingConfigurationItem.EntityColumnName);
if (propertyInfo != null)
{
var newMap = new CsvPropertyMap(propertyInfo);
newMap.TypeConverterOption(CultureInfo.InvariantCulture);
newMap.TypeConverterOption(NumberStyles.Any);
newMap.TypeConverterOption(true, "yes", "false");
//Map fields if mapping specified
if (mappingConfigurationItem.CsvColumnName != null)
{
newMap.Name(mappingConfigurationItem.CsvColumnName);
}
//And default values if it specified
if (mappingConfigurationItem.CustomValue != null)
{
newMap.Default(mappingConfigurationItem.CustomValue);
}
PropertyMaps.Add(newMap);
}
}
//Map Sku -> Code
Map(x => x.Code).ConvertUsing(x =>
{
return(GetCsvField("Sku", x, importConfiguration));
});
//Map ParentSku -> main product
Map(x => x.MainProductId).ConvertUsing(x =>
{
var parentSku = GetCsvField("ParentSku", x, importConfiguration);
return(!String.IsNullOrEmpty(parentSku) ? parentSku : null);
});
//Map assets (images)
Map(x => x.Assets).ConvertUsing(x =>
{
var retVal = new List <coreModel.ItemAsset>();
var primaryImageUrl = GetCsvField("PrimaryImage", x, importConfiguration);
var altImageUrl = GetCsvField("AltImage", x, importConfiguration);
if (!String.IsNullOrEmpty(primaryImageUrl))
{
retVal.Add(new coreModel.ItemAsset
{
Type = coreModel.ItemAssetType.Image,
Group = "primaryimage",
Url = primaryImageUrl
});
}
if (!String.IsNullOrEmpty(altImageUrl))
{
retVal.Add(new coreModel.ItemAsset
{
Type = coreModel.ItemAssetType.Image,
Group = "image",
Url = altImageUrl
});
}
return(retVal);
});
//Map category
Map(x => x.Category).ConvertUsing(x => new coreModel.Category {
Path = GetCsvField("Category", x, importConfiguration)
});
//Map Reviews
Map(x => x.Reviews).ConvertUsing(x =>
{
var reviews = new List <coreModel.EditorialReview>();
var content = GetCsvField("Review", x, importConfiguration);
if (!String.IsNullOrEmpty(content))
{
reviews.Add(new coreModel.EditorialReview {
Content = content, LanguageCode = defaultLanguge
});
}
return(reviews);
});
//Map Seo
Map(x => x.SeoInfos).ConvertUsing(x =>
{
var seoInfos = new List <SeoInfo>();
var seoUrl = GetCsvField("SeoUrl", x, importConfiguration);
var seoDescription = GetCsvField("SeoDescription", x, importConfiguration);
var seoTitle = GetCsvField("SeoTitle", x, importConfiguration);
if (!String.IsNullOrEmpty(seoUrl) || !String.IsNullOrEmpty(seoTitle) || !String.IsNullOrEmpty(seoDescription))
{
seoUrl = new string[] { seoUrl, seoDescription, seoTitle }.Where(y => !String.IsNullOrEmpty(y)).FirstOrDefault();
seoUrl = seoUrl.Substring(0, Math.Min(seoUrl.Length, 240));
seoInfos.Add(new SeoInfo {
LanguageCode = defaultLanguge, SemanticUrl = seoUrl.GenerateSlug(), MetaDescription = seoDescription, PageTitle = seoTitle
});
}
return(seoInfos);
});
//Map Prices
Map(x => x.Prices).ConvertUsing(x =>
{
var prices = new List <Price>();
var priceId = GetCsvField("PriceId", x, importConfiguration);
var listPrice = GetCsvField("Price", x, importConfiguration);
var salePrice = GetCsvField("SalePrice", x, importConfiguration);
var currency = GetCsvField("Currency", x, importConfiguration);
if (!String.IsNullOrEmpty(listPrice) && !String.IsNullOrEmpty(currency))
{
prices.Add(new Price
{
Id = priceId,
List = Convert.ToDecimal(listPrice, CultureInfo.InvariantCulture),
Sale = salePrice != null ? (decimal?)Convert.ToDecimal(listPrice, CultureInfo.InvariantCulture) : null,
Currency = EnumUtility.SafeParse <CurrencyCodes>(currency, CurrencyCodes.USD)
});
}
return(prices);
});
//Map inventories
Map(x => x.Inventories).ConvertUsing(x =>
{
var inventories = new List <InventoryInfo>();
var quantity = GetCsvField("Quantity", x, importConfiguration);
var allowBackorder = GetCsvField("AllowBackorder", x, importConfiguration);
var fulfilmentCenterId = GetCsvField("FulfilmentCenterId", x, importConfiguration);
if (!String.IsNullOrEmpty(quantity))
{
inventories.Add(new InventoryInfo
{
FulfillmentCenterId = fulfilmentCenterId,
AllowBackorder = allowBackorder.TryParse(false),
InStockQuantity = (long)quantity.TryParse(0.0m)
});
}
return(inventories);
});
//Map properties
if (importConfiguration.PropertyCsvColumns != null && importConfiguration.PropertyCsvColumns.Any())
{
Map(x => x.PropertyValues).ConvertUsing(x => importConfiguration.PropertyCsvColumns.Select(column => new coreModel.PropertyValue {
PropertyName = column, Value = x.GetField <string>(column)
}).ToList());
}
}
public IHttpActionResult DoImport(CsvImportConfiguration importConfiguration)
{
var notification = new ImportNotification(CurrentPrincipal.GetCurrentUserName())
{
Title = "Import catalog from CSV",
Description = "starting import...."
};
_notifier.Upsert(notification);
var importJob = new CsvCatalogImportJob();
BackgroundJob.Enqueue(() => importJob.DoImport(importConfiguration, notification));
return Ok(notification);
}
