public void CsvProductMapTest_CsvHasProductProperties_PropertiesMapped()
{
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.Delimiter = ",";
string path = @"../../data/product-productproperties.csv";
var csvProducts = ReadCsvFile(path, importInfo);
var product = csvProducts.FirstOrDefault();
Assert.Equal("429408", product.Id);
Assert.Equal("CBLK21113", product.Sku);
Assert.Equal("cblk21113-product-1", product.Name);
Assert.Equal("catId_1", product.CategoryId);
Assert.Equal("GTIN_Value", product.Gtin);
Assert.Equal("mainprod_123", product.MainProductId);
Assert.Equal("Vendor_value", product.Vendor);
Assert.Equal("ProductType_value", product.ProductType);
Assert.Equal("ShippingType_value", product.ShippingType);
Assert.Equal("DownloadType_value", product.DownloadType);
Assert.True(product.HasUserAgreement);
Assert.True(product.IsBuyable);
Assert.True(product.TrackInventory);
}
public void CsvProductMapTest_CsvHasProductProperties_PropertiesMapped()
{
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.Delimiter = ",";
string path = GetDataFilePath("product-productproperties.csv");
var csvProducts = ReadCsvFile(path, importInfo);
var product = csvProducts.FirstOrDefault();
Assert.Equal("429408", product.Id);
Assert.Equal("CBLK21113", product.Sku);
Assert.Equal("cblk21113-product-1", product.Name);
Assert.Equal("catId_1", product.CategoryId);
Assert.Equal("GTIN_Value", product.Gtin);
Assert.Equal("mainprod_123", product.MainProductId);
Assert.Equal("Vendor_value", product.Vendor);
Assert.Equal("ProductType_value", product.ProductType);
Assert.Equal("ShippingType_value", product.ShippingType);
Assert.Equal("DownloadType_value", product.DownloadType);
Assert.Equal("OuterId", product.OuterId);
Assert.Equal(1, product.Priority);
Assert.Equal(10, product.MaxQuantity);
Assert.Equal(5, product.MinQuantity);
Assert.Equal("PackageType", product.PackageType);
Assert.Equal("FulfillmentCenterId", product.FulfillmentCenterId);
Assert.Equal(1, product.MaxNumberOfDownload);
Assert.True(product.HasUserAgreement);
Assert.True(product.IsBuyable);
Assert.True(product.TrackInventory);
}
public void CsvProductMapTest_CsvHasPropertyValues_PropertyValuesMapped()
{
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.CsvColumns = new[] { "Sku" };
importInfo.Configuration.PropertyCsvColumns = new[] { "ProductProperty", "ProductProperty_Multivalue" };
importInfo.Configuration.Delimiter = ",";
string path = @"../../data/product-propertyvalues.csv";
var csvProducts = ReadCsvFile(path, importInfo);
Action <PropertyValue>[] inspectorsFirstProduct =
{
x => Assert.True((string)x.Value == "Product-1-propertyvalue-test" && x.PropertyName == "ProductProperty"),
x => Assert.True((string)x.Value == "Product-1-multivalue-1, Product-1-multivalue-2" && x.PropertyName == "ProductProperty_Multivalue")
};
Action <PropertyValue>[] inspectorsSecond =
{
x => Assert.True((string)x.Value == "Product-2-propertyvalue-test" && x.PropertyName == "ProductProperty"),
x => Assert.True((string)x.Value == "Product-2-multivalue-1, Product-2-multivalue-1, Product-2-multivalue-3" && x.PropertyName == "ProductProperty_Multivalue")
};
Assert.Collection(csvProducts.FirstOrDefault().PropertyValues, inspectorsFirstProduct);
Assert.Collection(csvProducts.LastOrDefault().PropertyValues, inspectorsSecond);
}
// Only public methods can be invoked in the background. (Hangfire)
public async Task BackgroundExport(CsvExportInfo exportInfo, ExportNotification notifyEvent)
{
var currencies = await _currencyService.GetAllCurrenciesAsync();
var defaultCurrency = currencies.First(x => x.IsPrimary);
exportInfo.Currency ??= defaultCurrency.Code;
var catalog = await _catalogService.GetByIdsAsync(new[] { exportInfo.CatalogId });
if (catalog == null)
{
throw new InvalidOperationException($"Cannot get catalog with id '{exportInfo.CatalogId}'");
}
Action <ExportImportProgressInfo> progressCallback = async x =>
{
notifyEvent.InjectFrom(x);
await _notifier.SendAsync(notifyEvent);
};
using (var stream = new MemoryStream())
{
try
{
exportInfo.Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration();
await _csvExporter.DoExportAsync(stream, exportInfo, progressCallback);
stream.Position = 0;
var fileNameTemplate = await _settingsManager.GetValueAsync(CsvModuleConstants.Settings.General.ExportFileNameTemplate.Name, CsvModuleConstants.Settings.General.ExportFileNameTemplate.DefaultValue.ToString());
var fileName = string.Format(fileNameTemplate, DateTime.UtcNow);
fileName = Path.ChangeExtension(fileName, ".csv");
var blobRelativeUrl = Path.Combine("temp", fileName);
//Upload result csv to blob storage
using (var blobStream = _blobStorageProvider.OpenWrite(blobRelativeUrl))
{
stream.CopyTo(blobStream);
}
//Get a download url
notifyEvent.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(blobRelativeUrl);
notifyEvent.Description = "Export finished";
}
catch (Exception ex)
{
notifyEvent.Description = "Export failed";
notifyEvent.Errors.Add(ex.ExpandExceptionMessage());
}
finally
{
notifyEvent.Finished = DateTime.UtcNow;
await _notifier.SendAsync(notifyEvent);
}
}
}
public void BackgroundExport(CsvExportInfo exportInfo, ExportNotification notifyEvent)
{
var curencySetting = _settingsManager.GetSettingByName("VirtoCommerce.Core.General.Currencies");
var defaultCurrency = EnumUtility.SafeParse <CurrencyCodes>(curencySetting.DefaultValue, CurrencyCodes.USD);
exportInfo.Currency = exportInfo.Currency ?? defaultCurrency;
var catalog = _catalogService.GetById(exportInfo.CatalogId);
if (catalog == null)
{
throw new NullReferenceException("catalog");
}
Action <ExportImportProgressInfo> progressCallback = (x) =>
{
notifyEvent.InjectFrom(x);
_notifier.Upsert(notifyEvent);
};
using (var stream = new MemoryStream())
{
try
{
exportInfo.Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration();
_csvExporter.DoExport(stream, exportInfo, progressCallback);
stream.Position = 0;
//Upload result csv to blob storage
var uploadInfo = new UploadStreamInfo
{
FileName = "Catalog-" + catalog.Name + "-export.csv",
FileByteStream = stream,
FolderName = "temp"
};
var blobKey = _blobStorageProvider.Upload(uploadInfo);
//Get a download url
notifyEvent.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(blobKey);
notifyEvent.Description = "Export finished";
}
catch (Exception ex)
{
notifyEvent.Description = "Export failed";
notifyEvent.Errors.Add(ex.ExpandExceptionMessage());
}
finally
{
notifyEvent.Finished = DateTime.UtcNow;
_notifier.Upsert(notifyEvent);
}
}
}
public void CsvProductMapTest_CsvHasMultipleLines_LineNumberMapTest()
{
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.Delimiter = ",";
string path = @"../../data/product-productproperties-twoproducts.csv";
var csvProducts = ReadCsvFile(path, importInfo);
Assert.Equal(2, csvProducts[0].LineNumber);
Assert.Equal(3, csvProducts[1].LineNumber);
}
// Only public methods can be invoked in the background. (Hangfire)
public void BackgroundExport(CsvExportInfo exportInfo, ExportNotification notifyEvent)
{
var currencies = _commerceService.GetAllCurrencies();
var defaultCurrency = currencies.First(x => x.IsPrimary);
exportInfo.Currency = exportInfo.Currency ?? defaultCurrency.Code;
var catalog = _catalogService.GetById(exportInfo.CatalogId);
if (catalog == null)
{
throw new NullReferenceException("catalog");
}
Action <ExportImportProgressInfo> progressCallback = x =>
{
notifyEvent.InjectFrom(x);
_notifier.Upsert(notifyEvent);
};
using (var stream = new MemoryStream())
{
try
{
exportInfo.Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration();
_csvExporter.DoExport(stream, exportInfo, progressCallback);
stream.Position = 0;
var blobRelativeUrl = "temp/Catalog-" + catalog.Name + "-export.csv";
//Upload result csv to blob storage
using (var blobStream = _blobStorageProvider.OpenWrite(blobRelativeUrl))
{
stream.CopyTo(blobStream);
}
//Get a download url
notifyEvent.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(blobRelativeUrl);
notifyEvent.Description = "Export finished";
}
catch (Exception ex)
{
notifyEvent.Description = "Export failed";
notifyEvent.Errors.Add(ex.ExpandExceptionMessage());
}
finally
{
notifyEvent.Finished = DateTime.UtcNow;
_notifier.Upsert(notifyEvent);
}
}
}
public void CsvProductMapTest_CsvHasCategorypath_CategoryPathMapped()
{
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.Delimiter = ",";
string path = @"../../data/product-productproperties-categoryPath.csv";
var csvProducts = ReadCsvFile(path, importInfo);
var product = csvProducts.FirstOrDefault();
Assert.Equal("TestCategory1", product.CategoryPath);
Assert.Equal("TestCategory1", product.Category.Path);
}
public void CsvProductMapTest_CsvHasReview_ReviewMapped()
{
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.Delimiter = ",";
string path = @"../../data/product-productproperties-review.csv";
var csvProducts = ReadCsvFile(path, importInfo);
var product = csvProducts.FirstOrDefault();
Assert.Equal("Review_Content", product.Review);
Assert.Equal("Review_Content", product.EditorialReview.Content);
Assert.Equal("ReviewType_Value", product.ReviewType);
Assert.Equal("ReviewType_Value", product.EditorialReview.ReviewType);
}
public IHttpActionResult GetMappingConfiguration([FromUri] string fileUrl, [FromUri] string delimiter = ";")
{
var retVal = CsvProductMappingConfiguration.GetDefaultConfiguration();
retVal.Delimiter = delimiter;
//Read csv headers and try to auto map fields by name
using (var reader = new CsvReader(new StreamReader(_blobStorageProvider.OpenRead(fileUrl))))
{
reader.Configuration.Delimiter = delimiter;
if (reader.Read())
{
retVal.AutoMap(reader.FieldHeaders);
}
}
return(Ok(retVal));
}
public void CsvProductMapTest_CsvHasBooleanValues_BooleanFieldsMapped()
{
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.Delimiter = ",";
string path = @"../../data/product-productproperties-boolean.csv";
var csvProducts = ReadCsvFile(path, importInfo);
Assert.False(csvProducts[0].HasUserAgreement);
Assert.False(csvProducts[0].IsBuyable);
Assert.False(csvProducts[0].TrackInventory);
Assert.True(csvProducts[1].HasUserAgreement);
Assert.True(csvProducts[1].IsBuyable);
Assert.True(csvProducts[1].TrackInventory);
}
public void CsvHeadersExportTest_DefaultConfiguration_HeadersAreSame()
{
using (var sw = new StringWriter())
{
using (var csv = new CsvWriter(sw))
{
CsvExportInfo exportInfo = new CsvExportInfo();
exportInfo.Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration();
csv.Configuration.Delimiter = exportInfo.Configuration.Delimiter;
csv.Configuration.RegisterClassMap(new CsvProductMap(exportInfo.Configuration));
csv.WriteHeader <CsvProduct>();
csv.Flush();
var expected = string.Join(exportInfo.Configuration.Delimiter, exportInfo.Configuration.PropertyMaps.Select(x => x.CsvColumnName));
Assert.Equal(expected, sw.ToString());
}
}
}
public async Task <ActionResult <CsvProductMappingConfiguration> > GetMappingConfiguration([FromQuery] string fileUrl, [FromQuery] string delimiter = ";")
{
var result = CsvProductMappingConfiguration.GetDefaultConfiguration();
var decodedDelimiter = HttpUtility.UrlDecode(delimiter);
result.Delimiter = decodedDelimiter;
//Read csv headers and try to auto map fields by name
using (var reader = new CsvReader(new StreamReader(_blobStorageProvider.OpenRead(fileUrl))))
{
reader.Configuration.Delimiter = decodedDelimiter;
if (await reader.ReadAsync() && reader.ReadHeader())
{
result.AutoMap(reader.Context.HeaderRecord);
}
}
return(Ok(result));
}
public void CsvProductMapTest_CsvHasSeoInfo_SeoInfoMapped()
{
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.Delimiter = ",";
string path = @"../../data/product-productproperties-seoInfo.csv";
var csvProducts = ReadCsvFile(path, importInfo);
var product = csvProducts.FirstOrDefault(x => x.Code == "cblk21113-product-1");
Assert.Equal("seo-slug-url", product.SeoUrl);
Assert.Equal("seo-slug-url", product.SeoInfo.SemanticUrl);
Assert.Equal("Seo_Title_Value", product.SeoTitle);
Assert.Equal("Seo_Title_Value", product.SeoInfo.PageTitle);
Assert.Equal("Seo_Descr_Value", product.SeoDescription);
Assert.Equal("Seo_Descr_Value", product.SeoInfo.MetaDescription);
Assert.Equal("Seo_Language_Value", product.SeoInfo.LanguageCode);
Assert.True(csvProducts.Count == 2);
}
public void CsvProductMapTest_CsvHasPriceAndQuantity_PriceAndQuantityMapped()
{
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.Delimiter = ",";
string path = @"../../data/product-productproperties-priceQuantity.csv";
var csvProducts = ReadCsvFile(path, importInfo);
var product = csvProducts.FirstOrDefault();
Assert.Equal("123.4", product.ListPrice);
Assert.Equal(123.4m, product.Price.List);
Assert.Equal("456.7", product.SalePrice);
Assert.Equal(456.7m, product.Price.Sale);
Assert.Equal("EUR", product.Currency);
Assert.Equal("EUR", product.Price.Currency);
Assert.Equal("5", product.PriceMinQuantity);
Assert.Equal(5, product.Price.MinQuantity);
}
public void CsvProductMapTest_MappingHasDefaultBoolValue_DefaultBoolValuesMapped()
{
var defaultValue = true;
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.Delimiter = ",";
var categoryPathMapping = importInfo.Configuration.PropertyMaps.FirstOrDefault(x => x.EntityColumnName == "IsBuyable");
categoryPathMapping.CsvColumnName = null;
categoryPathMapping.CustomValue = defaultValue.ToString();
string path = @"../../data/product-productproperties.csv";
var csvProducts = ReadCsvFile(path, importInfo);
var product = csvProducts.FirstOrDefault();
Assert.True(product.IsBuyable);
}
private CsvProduct ExportAndImportProduct(CatalogProduct product)
{
var exportInfo = new CsvExportInfo();
using (var stream = new MemoryStream())
{
var streamWriter = new StreamWriter(stream, Encoding.UTF8, 1024, true)
{
AutoFlush = true
};
using (var csvWriter = new CsvWriter(streamWriter))
{
exportInfo.Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration();
exportInfo.Configuration.PropertyCsvColumns = product.Properties.Select(x => x.Name).Distinct().ToArray();
csvWriter.Configuration.Delimiter = exportInfo.Configuration.Delimiter;
csvWriter.Configuration.RegisterClassMap(new CsvProductMap(exportInfo.Configuration));
csvWriter.WriteHeader <CsvProduct>();
csvWriter.NextRecord();
var csvProduct = new CsvProduct(product, null, null, null, null);
csvWriter.WriteRecord(csvProduct);
csvWriter.Flush();
stream.Position = 0;
}
using (var reader = new CsvReader(new StreamReader(stream, Encoding.UTF8)))
{
reader.Configuration.Delimiter = exportInfo.Configuration.Delimiter;
reader.Configuration.RegisterClassMap(new CsvProductMap(exportInfo.Configuration));
reader.Configuration.MissingFieldFound = (strings, i, arg3) =>
{
//do nothing
};
reader.Configuration.TrimOptions = TrimOptions.Trim;
reader.Read();
return(reader.GetRecord <CsvProduct>());
}
}
}
public void CsvProductMapTest_MappingHasDefaultCategoryPath_DefaultCategoryPathMapped()
{
var defaultValue = "Custom_category_path_value";
var importInfo = new CsvImportInfo {
Configuration = CsvProductMappingConfiguration.GetDefaultConfiguration()
};
importInfo.Configuration.Delimiter = ",";
var categoryPathMapping = importInfo.Configuration.PropertyMaps.FirstOrDefault(x => x.EntityColumnName == "CategoryPath");
categoryPathMapping.CsvColumnName = null;
categoryPathMapping.CustomValue = defaultValue;
string path = @"../../data/product-productproperties-noCategoryPath.csv";
var csvProducts = ReadCsvFile(path, importInfo);
var product = csvProducts.FirstOrDefault();
Assert.Equal(defaultValue, product.CategoryPath);
Assert.Equal(defaultValue, product.Category.Path);
}
public CsvProductMap(CsvProductMappingConfiguration mappingCfg)
{
//Dynamical map scalar product fields use by manual mapping information
var index = 0;
foreach (var mappingItem in mappingCfg.PropertyMaps.Where(x => !string.IsNullOrEmpty(x.CsvColumnName) || !string.IsNullOrEmpty(x.CustomValue)))
{
var propertyInfo = typeof(CsvProduct).GetProperty(mappingItem.EntityColumnName);
if (propertyInfo != null)
{
var newMap = MemberMap.CreateGeneric(typeof(CsvProduct), propertyInfo);
newMap.Data.TypeConverterOptions.CultureInfo = CultureInfo.InvariantCulture;
newMap.Data.TypeConverterOptions.NumberStyle = NumberStyles.Any;
newMap.Data.TypeConverterOptions.BooleanTrueValues.AddRange(new List <string>()
{
"yes", "true"
});
newMap.Data.TypeConverterOptions.BooleanFalseValues.AddRange(new List <string>()
{
"false", "no"
});
newMap.Data.Index = ++index;
if (!string.IsNullOrEmpty(mappingItem.CsvColumnName))
{
//Map fields if mapping specified
newMap.Name(mappingItem.CsvColumnName);
}
//And default values if it specified
else if (mappingItem.CustomValue != null)
{
var typeConverter = TypeDescriptor.GetConverter(propertyInfo.PropertyType);
newMap.Data.ReadingConvertExpression = (Expression <Func <IReaderRow, object> >)(x => typeConverter.ConvertFromString(mappingItem.CustomValue));
newMap.Default(mappingItem.CustomValue);
}
MemberMaps.Add(newMap);
}
}
//Map properties
if (mappingCfg.PropertyCsvColumns != null && mappingCfg.PropertyCsvColumns.Any())
{
// Exporting multiple csv fields from the same property (which is a collection)
foreach (var propertyCsvColumn in mappingCfg.PropertyCsvColumns)
{
// create CsvPropertyMap manually, because this.Map(x =>...) does not allow
// to export multiple entries for the same property
var propertyValuesInfo = typeof(CsvProduct).GetProperty(nameof(CsvProduct.Properties));
var csvPropertyMap = MemberMap.CreateGeneric(typeof(CsvProduct), propertyValuesInfo);
csvPropertyMap.Name(propertyCsvColumn);
csvPropertyMap.Data.Index = ++index;
// create custom converter instance which will get the required record from the collection
csvPropertyMap.UsingExpression <ICollection <Property> >(null, properties =>
{
var property = properties.FirstOrDefault(x => x.Name == propertyCsvColumn && x.Values.Any());
var propertyValues = Array.Empty <string>();
if (property != null)
{
if (property.Dictionary)
{
propertyValues = property.Values
?.Where(x => !string.IsNullOrEmpty(x.Alias))
.Select(x => x.Alias)
.Distinct()
.ToArray();
}
else
{
propertyValues = property.Values
?.Where(x => x.Value != null || x.Alias != null)
.Select(x => x.Alias ?? x.Value.ToString())
.ToArray();
}
}
return(string.Join(mappingCfg.Delimiter, propertyValues));
});
MemberMaps.Add(csvPropertyMap);
}
var newPropInfo = typeof(CsvProduct).GetProperty(nameof(CsvProduct.Properties));
var newPropMap = MemberMap.CreateGeneric(typeof(CsvProduct), newPropInfo);
newPropMap.Data.ReadingConvertExpression =
(Expression <Func <IReaderRow, object> >)(x => mappingCfg.PropertyCsvColumns.Select(column =>
(Property) new CsvProperty
{
Name = column,
Values = new List <PropertyValue>()
{
new PropertyValue()
{
PropertyName = column,
Value = x.GetField <string>(column)
}
}
}).ToList());
newPropMap.UsingExpression <ICollection <PropertyValue> >(null, null);
newPropMap.Data.Index = ++index;
MemberMaps.Add(newPropMap);
newPropMap.Ignore(true);
}
//map line number
var lineNumMeber = Map(m => m.LineNumber).ConvertUsing(row => row.Context.RawRow);
lineNumMeber.Data.Index = ++index;
lineNumMeber.Ignore(true);
}