private void config <T>(CsvConfiguration csvConfig, Configuracao config)
{
config = config == null ? new Configuracao() : config;
csvConfig.Delimiter = config.Delimitador;
csvConfig.IgnoreHeaderWhiteSpace = config.IgnoreEspacoEmBrancoHeader;
csvConfig.TrimFields = config.RemoverEspacoEmBrancoCampos;
if (config.Map.Count > 0)
{
var customerMap = new CsvHelper.Configuration.DefaultCsvClassMap <T>();
var sampleObject = Activator.CreateInstance <T>();
foreach (var map in config.Map)
{
var propertyInfo = sampleObject.GetType().GetProperty(map.NomeDaPropriedade);
var newMap = new CsvPropertyMap(propertyInfo);
newMap.Name(map.NomeDaColuna);
if (map.Converter != null)
{
newMap.ConvertUsing(
row => map.Converter.Conversao(row.GetField <string>(map.Converter.NomeDaColuna))
);
}
customerMap.PropertyMaps.Add(newMap);
}
csvConfig.RegisterClassMap(customerMap);
}
}
public List <T> DeserializeList <T>(string list)
{
if (list != "")
{
PropertyInfo[] properties = typeof(T).GetRuntimeProperties().Where(x => !x.SetMethod.IsVirtual).ToArray();
CsvClassMap map = new DefaultCsvClassMap <T>();
foreach (PropertyInfo pi in properties)
{
CsvPropertyMap propMap = new CsvPropertyMap(pi);
propMap.Name(pi.Name);
propMap.Index(0);
map.PropertyMaps.Add(propMap);
}
TextReader tr = new StringReader(list);
CsvReader reader = new CsvReader(tr);
reader.Configuration.Delimiter = delimiter;
reader.Configuration.RegisterClassMap(map);
List <T> result = new List <T>();
foreach (T record in reader.GetRecords <T>())
{
result.Add(record);
}
return(result);
}
return(new List <T>());
}
public CsvProductMap(CsvProductMappingConfiguration mappingCfg)
{
//Dynamical map scalar product fields use by manual mapping information
foreach (var mappingItem in mappingCfg.PropertyMaps.Where(x => !string.IsNullOrEmpty(x.CsvColumnName) || !string.IsNullOrEmpty(x.CustomValue)))
{
var propertyInfo = typeof(CsvProduct).GetProperty(mappingItem.EntityColumnName);
if (propertyInfo != null)
{
var newMap = new CsvPropertyMap(propertyInfo);
newMap.TypeConverterOption(CultureInfo.InvariantCulture);
newMap.TypeConverterOption(NumberStyles.Any);
newMap.TypeConverterOption(true, "yes", "true");
newMap.TypeConverterOption(false, "false", "no");
if (!string.IsNullOrEmpty(mappingItem.CsvColumnName))
{
//Map fields if mapping specified
newMap.Name(mappingItem.CsvColumnName);
}
//And default values if it specified
if (mappingItem.CustomValue != null)
{
var typeConverter = TypeDescriptor.GetConverter(propertyInfo.PropertyType);
newMap.Default(typeConverter.ConvertFromString(mappingItem.CustomValue));
}
PropertyMaps.Add(newMap);
}
}
//Map properties
if (mappingCfg.PropertyCsvColumns != null && mappingCfg.PropertyCsvColumns.Any())
{
// Exporting multiple csv fields from the same property (which is a collection)
foreach (var propertyCsvColumn in mappingCfg.PropertyCsvColumns)
{
// create CsvPropertyMap manually, because this.Map(x =>...) does not allow
// to export multiple entries for the same property
var csvPropertyMap = new CsvPropertyMap(typeof(CsvProduct).GetProperty("PropertyValues"));
csvPropertyMap.Name(propertyCsvColumn);
// create custom converter instance which will get the required record from the collection
csvPropertyMap.UsingExpression <ICollection <coreModel.PropertyValue> >(null, propValues =>
{
var propValue = propValues.FirstOrDefault(x => x.PropertyName == propertyCsvColumn);
if (propValue != null)
{
return(propValue.Value != null ? propValue.Value.ToString() : string.Empty);
}
return(string.Empty);
});
PropertyMaps.Add(csvPropertyMap);
}
var newPropMap = new CsvPropertyMap(typeof(CsvProduct).GetProperty("PropertyValues"));
newPropMap.UsingExpression <ICollection <coreModel.PropertyValue> >(null, null).ConvertUsing(x => mappingCfg.PropertyCsvColumns.Select(column => new coreModel.PropertyValue {
PropertyName = column, Value = x.GetField <string>(column)
}).ToList());
PropertyMaps.Add(newPropMap);
}
}
public string SerializeList <T>(List <T> list)
{
Type t = typeof(T);
PropertyInfo[] properties = t.GetRuntimeProperties().Where(x => !x.SetMethod.IsVirtual).ToArray();
CsvClassMap map = new DefaultCsvClassMap <T>();
foreach (PropertyInfo pi in properties)
{
CsvPropertyMap propMap = new CsvPropertyMap(pi);
propMap.Name(pi.Name);
propMap.Index(0);
map.PropertyMaps.Add(propMap);
}
StringWriter sw = new StringWriter();
CsvWriter writer = new CsvWriter(sw);
writer.Configuration.Delimiter = delimiter;
writer.Configuration.RegisterClassMap(map);
writer.WriteRecords(list);
return(sw.ToString());
}
public async Task <List <T> > ReadRecords <T>(IS3Path path, Func <T, bool> predicate, bool isOptional = false)
{
_logger.Information("Reading {Type} records from {@S3Path}", typeof(T).Name, path);
var configuration = new CsvConfiguration
{
Delimiter = "|",
QuoteNoFields = true,
HasHeaderRecord = true
};
var customerMap = new DefaultCsvClassMap <T>();
foreach (var prop in typeof(T).GetProperties())
{
var name = prop.Name.ToSnakeCase();
var map = new CsvPropertyMap(prop).Name(name);
customerMap.PropertyMaps.Add(map);
}
configuration.RegisterClassMap(customerMap);
try
{
using (var response = await _s3Client.GetObjectAsync(new GetObjectRequest {
BucketName = path.BucketName, Key = path.Key
}))
{
using (var responseStream = response.ResponseStream)
{
using (var streamReader = new StreamReader(responseStream))
{
using (var reader = new CsvReader(streamReader, configuration))
{
var results = reader.GetRecords <T>();
if (predicate != null)
{
results = results.Where(predicate);
}
return(results.ToList());
}
}
}
}
}
catch (AmazonS3Exception ex)
{
if (ex.ErrorCode == "NoSuchKey" && isOptional)
{
return(new List <T>());
}
throw;
}
catch (Exception e)
{
_logger.Fatal("Exception when trying to read csv file: {@Message}", e.Message);
throw;
}
}
public static CsvPropertyMap Field <T>(this CsvPropertyMap source, string name, Func <string, T> modifier)
{
return(source.ConvertUsing(row =>
{
string value = row.GetField(name);
return modifier(value);
}));
}
public DataImportClient()
{
/*
* groups = fields.GroupBy(
* f => f.Substring(
* 0, f.IndexOf('.')
* ), f => f.Remove(
* 0, f.IndexOf('.') + 1
* )
* ).Select(g => new Group(
* g.Key, g.ToArray()
* )).OrderBy(g => g.prefix).ToArray();*/
cfg = new CsvConfiguration();
cfg.Delimiter = "\t";
cfg.DetectColumnCountChanges = true;
cfg.TrimFields = true;
//cfg.RegisterClassMap(typeof(Group));
csvMapping = (Lookup <string, Lookup <string, string> >)fields.ToLookup(
f => f.Substring(0, f.IndexOf('.')),
f => (Lookup <string, string>)Enumerable
.Empty <string>().ToLookup(
p => f.Remove(
0, f.IndexOf('.'))));
/* var csvMap = new DefaultCsvClassMap<Lookup<string, string>>();
* foreach(var group in csvMapping)
* {
* foreach(var subgroup in group)
* {
* foreach(var item in subgroup)
* {
* var newMap = new CsvPropertyMap(
* typeof(IGrouping<string, string>).GetType().GetProperty(item.Key)
* );
* }
* }
* }*/
fileFormatMap = new DefaultCsvClassMap <FileFormatSkeleton>();
Type ffs = typeof(FileFormatSkeleton);
ffsFields = ffs.GetFields(BindingFlags.SetProperty).ToDictionary(f => f.Name, f => f);
var fieldNames = fields.Select(f => new KeyValuePair <string, string>(f, f.Replace('.', '_')));
foreach (var field in fieldNames)
{
var newMap = new CsvPropertyMap(
ffs.GetProperty(field.Value));
newMap.Name(field.Key);
fileFormatMap.PropertyMaps.Add(newMap);
}
cfg.RegisterClassMap(fileFormatMap);
}
public static CsvPropertyMap Required <T>(this CsvPropertyMap map, string columnName)
{
return(map.Name(columnName).ConvertUsing(row =>
{
if (string.IsNullOrEmpty(row.GetField(columnName)))
{
throw new CsvParserException($"{columnName} is required, but missing from row {row.Row}");
}
return row.GetField <T>(columnName);
}));
}
/// <summary>
/// Determines if the property for the <see cref="CsvPropertyMap"/>
/// can be read.
/// </summary>
/// <param name="propertyMap">The property map.</param>
/// <returns>A value indicating of the property can be read. True if it can, otherwise false.</returns>
protected bool CanRead(CsvPropertyMap propertyMap)
{
var cantRead =
// Ignored properties.
propertyMap.IgnoreValue ||
// Properties that don't have a public setter
// and we are honoring the accessor modifier.
propertyMap.PropertyValue.GetSetMethod() == null && !configuration.IgnorePrivateAccessor ||
// Properties that don't have a setter at all.
propertyMap.PropertyValue.GetSetMethod(true) == null;
return(!cantRead);
}
/// <summary>
/// Checks if the property can be written.
/// </summary>
/// <param name="propertyMap">The property map that we are checking.</param>
/// <returns>A value indicating if the property can be written.
/// True if the property can be written, otherwise false.</returns>
protected virtual bool CanWrite(CsvPropertyMap propertyMap)
{
var cantWrite =
// Ignored properties.
propertyMap.Data.Ignore ||
// Properties that don't have a public getter
// and we are honoring the accessor modifier.
propertyMap.Data.Property.GetGetMethod() == null && !configuration.IgnorePrivateAccessor ||
// Properties that don't have a getter at all.
propertyMap.Data.Property.GetGetMethod(true) == null;
return(!cantWrite);
}
public ImportedEventNameClassMap()
{
// allow each field to be serialized by PascalCase, snake_case, or camelCase
foreach (var property in Properties)
{
var map = new CsvPropertyMap(property);
var pascalName = property.Name;
var snakeName = pascalName.ToSnakeCase();
var camelName = pascalName.ToCamelCase();
map.Name(pascalName, snakeName, camelName);
this.PropertyMaps.Add(map);
}
}
/// <summary>
/// Checks if the property/field can be written.
/// </summary>
/// <param name="propertyMap">The property/field map that we are checking.</param>
/// <returns>A value indicating if the property/field can be written.
/// True if the property/field can be written, otherwise false.</returns>
protected virtual bool CanWrite(CsvPropertyMap propertyMap)
{
var cantWrite =
// Ignored properties/fields.
propertyMap.Data.Ignore;
var property = propertyMap.Data.Member as PropertyInfo;
if (property != null)
{
cantWrite = cantWrite ||
// Properties that don't have a public getter
// and we are honoring the accessor modifier.
property.GetGetMethod() == null && !configuration.IncludePrivateMembers ||
// Properties that don't have a getter at all.
property.GetGetMethod(true) == null;
}
return(!cantWrite);
}
public ProductMap(string[] allColumns, webModel.CsvImportMappingItem[] mappingConfiguration)
{
var attributePropertyNames = allColumns.Except(mappingConfiguration.Select(x => x.CsvColumnName));
foreach (var mappingConfigurationItem in mappingConfiguration)
{
var propertyInfo = typeof(coreModel.CatalogProduct).GetProperty(mappingConfigurationItem.EntityColumnName);
var newMap = new CsvPropertyMap(propertyInfo);
newMap.Name(mappingConfigurationItem.CsvColumnName);
PropertyMaps.Add(newMap);
}
var categoryMappingItem = mappingConfiguration.First(x => x.EntityColumnName == "Category");
var editorialReviewMappingItem = mappingConfiguration.First(x => x.EntityColumnName == "Reviews");
Map(x => x.Category).ConvertUsing(x => new coreModel.Category {
Path = x.GetField <string>(categoryMappingItem.CsvColumnName)
});
Map(x => x.Reviews).ConvertUsing(x => new coreModel.EditorialReview[] { new coreModel.EditorialReview {
Content = x.GetField <string>(editorialReviewMappingItem.CsvColumnName)
} });
Map(x => x.PropertyValues).ConvertUsing(x => attributePropertyNames.Select(column => new coreModel.PropertyValue {
PropertyName = column, Value = x.GetField <string>(column)
}).ToList());
}
/// <summary>
/// Creates a parameter for the given property. This will
/// recursively traverse the mapping to to find property
/// mapping and create a new property access for each
/// reference map it goes through.
/// </summary>
/// <param name="parameter">The current parameter.</param>
/// <param name="mapping">The mapping to look for the property map on.</param>
/// <param name="propertyMap">The property map to look for on the mapping.</param>
/// <returns>A <see cref="ParameterExpression"/> to access the given property map.</returns>
protected virtual Expression CreateParameterForProperty(Expression parameter, CsvClassMap mapping, CsvPropertyMap propertyMap)
{
var propertyMapping = mapping.PropertyMaps.SingleOrDefault(pm => pm == propertyMap);
if (propertyMapping != null)
{
// If the property map exists on this level of the class
// mapping, we can return the parameter.
return(parameter);
}
// The property isn't on this level of the mapping.
// We need to search down through the reference maps.
foreach (var refMap in mapping.ReferenceMaps)
{
var wrappedParameter = Expression.Property(parameter, refMap.Property);
var param = CreateParameterForProperty(wrappedParameter, refMap.Mapping, propertyMap);
if (param != null)
{
return(param);
}
}
return(null);
}
public static CsvPropertyMap UsingExpression <T>(this CsvPropertyMap map, Func <string, T> readExpression,
Func <T, string> writeExpression)
{
return(map.TypeConverter(new ExpressionConverter <T>(readExpression, writeExpression)));
}
/// <summary>
/// Creates a property expression for the given property on the record.
/// This will recursively traverse the mapping to find the property
/// and create a safe property accessor for each level as it goes.
/// </summary>
/// <param name="recordExpression">The current property expression.</param>
/// <param name="mapping">The mapping to look for the property to map on.</param>
/// <param name="propertyMap">The property map to look for on the mapping.</param>
/// <returns>An Expression to access the given property.</returns>
protected virtual Expression CreatePropertyExpression(Expression recordExpression, CsvClassMap mapping, CsvPropertyMap propertyMap)
{
if (mapping.PropertyMaps.Any(pm => pm == propertyMap))
{
// The property is on this level.
return(Expression.Property(recordExpression, propertyMap.Data.Property));
}
// The property isn't on this level of the mapping.
// We need to search down through the reference maps.
foreach (var refMap in mapping.ReferenceMaps)
{
var wrapped = Expression.Property(recordExpression, refMap.Data.Property);
var propertyExpression = CreatePropertyExpression(wrapped, refMap.Data.Mapping, propertyMap);
if (propertyExpression == null)
{
continue;
}
if (refMap.Data.Property.PropertyType.GetTypeInfo().IsValueType)
{
return(propertyExpression);
}
var nullCheckExpression = Expression.Equal(wrapped, Expression.Constant(null));
var isValueType = propertyMap.Data.Property.PropertyType.GetTypeInfo().IsValueType;
var isGenericType = isValueType && propertyMap.Data.Property.PropertyType.GetTypeInfo().IsGenericType;
Type propertyType;
if (isValueType && !isGenericType && !configuration.UseNewObjectForNullReferenceProperties)
{
propertyType = typeof(Nullable <>).MakeGenericType(propertyMap.Data.Property.PropertyType);
propertyExpression = Expression.Convert(propertyExpression, propertyType);
}
else
{
propertyType = propertyMap.Data.Property.PropertyType;
}
var defaultValueExpression = isValueType && !isGenericType
? (Expression)Expression.New(propertyType)
: Expression.Constant(null, propertyType);
var conditionExpression = Expression.Condition(nullCheckExpression, defaultValueExpression, propertyExpression);
return(conditionExpression);
}
return(null);
}
/// <summary>
/// Maps a non-member to a CSV field. This allows for writing
/// data that isn't mapped to a class property/field.
/// </summary>
/// <returns>The property mapping.</returns>
public virtual CsvPropertyMap Map()
{
var propertyMap = new CsvPropertyMap( null );
propertyMap.Data.Index = GetMaxIndex() + 1;
PropertyMaps.Add( propertyMap );
return propertyMap;
}
/// <summary>
/// Creates a property expression for the given property on the record.
/// This will recursively traverse the mapping to find the property
/// and create a safe property accessor for each level as it goes.
/// </summary>
/// <param name="recordExpression">The current property expression.</param>
/// <param name="mapping">The mapping to look for the property to map on.</param>
/// <param name="propertyMap">The property map to look for on the mapping.</param>
/// <returns>An Expression to access the given property.</returns>
protected virtual Expression CreatePropertyExpression(Expression recordExpression, CsvClassMap mapping, CsvPropertyMap propertyMap)
{
if (mapping.PropertyMaps.Any(pm => pm == propertyMap))
{
// The property is on this level.
return(Expression.Property(recordExpression, propertyMap.Data.Property));
}
// The property isn't on this level of the mapping.
// We need to search down through the reference maps.
foreach (var refMap in mapping.ReferenceMaps)
{
var wrapped = Expression.Property(recordExpression, refMap.Property);
var propertyExpression = CreatePropertyExpression(wrapped, refMap.Mapping, propertyMap);
if (propertyExpression == null)
{
continue;
}
var isReferenceValueType = refMap.Property.PropertyType.IsValueType;
if (isReferenceValueType)
{
return(propertyExpression);
}
var nullCheckExpression = Expression.Equal(wrapped, Expression.Constant(null));
var isValueType = propertyMap.Data.Property.PropertyType.IsValueType;
var defaultValueExpression = isValueType
? (Expression)Expression.New(propertyMap.Data.Property.PropertyType)
: Expression.Constant(null, propertyMap.Data.Property.PropertyType);
var conditionExpression = Expression.Condition(nullCheckExpression, defaultValueExpression, propertyExpression);
return(conditionExpression);
}
return(null);
}
/// <summary>
/// Maps a property/field to a CSV field.
/// </summary>
/// <param name="member">The property/field to map.</param>
/// <param name="useExistingMap">If true, an existing map will be used if available.
/// If false, a new map is created for the same property/field.</param>
/// <returns>The property/field mapping.</returns>
public virtual CsvPropertyMap Map( MemberInfo member, bool useExistingMap = true )
{
if( useExistingMap )
{
var existingMap = PropertyMaps.Find( member );
if( existingMap != null )
{
return existingMap;
}
}
var propertyMap = new CsvPropertyMap( member );
propertyMap.Data.Index = GetMaxIndex() + 1;
PropertyMaps.Add( propertyMap );
return propertyMap;
}
/// <summary>
/// Auto maps the given map and checks for circular references as it goes.
/// </summary>
/// <param name="map">The map to auto map.</param>
/// <param name="options">Options for auto mapping.</param>
/// <param name="mapParents">The list of parents for the map.</param>
/// <param name="indexStart">The index starting point.</param>
internal static void AutoMapInternal( CsvClassMap map, AutoMapOptions options, LinkedList<Type> mapParents, int indexStart = 0 )
{
var type = map.GetType().GetTypeInfo().BaseType.GetGenericArguments()[0];
if( typeof( IEnumerable ).IsAssignableFrom( type ) )
{
throw new CsvConfigurationException( "Types that inherit IEnumerable cannot be auto mapped. " +
"Did you accidentally call GetRecord or WriteRecord which " +
"acts on a single record instead of calling GetRecords or " +
"WriteRecords which acts on a list of records?" );
}
var flags = BindingFlags.Instance | BindingFlags.Public;
if( options.IncludePrivateProperties )
{
flags = flags | BindingFlags.NonPublic;
}
var members = new List<MemberInfo>();
if( ( options.MemberTypes & MemberTypes.Properties ) == MemberTypes.Properties )
{
var properties = type.GetProperties( flags );
members.AddRange( properties );
}
if( ( options.MemberTypes & MemberTypes.Fields ) == MemberTypes.Fields )
{
var fields = new List<MemberInfo>();
foreach( var field in type.GetFields( flags ) )
{
if( !field.GetCustomAttributes( typeof( CompilerGeneratedAttribute ), false ).Any() )
{
fields.Add( field );
}
}
members.AddRange( fields );
}
foreach( var member in members )
{
var typeConverterType = TypeConverterFactory.GetConverter( member.MemberType() ).GetType();
if( typeConverterType == typeof( EnumerableConverter ) )
{
// The IEnumerable converter just throws an exception so skip it.
continue;
}
var isDefaultConverter = typeConverterType == typeof( DefaultTypeConverter );
var hasDefaultConstructor = member.MemberType().GetConstructor( new Type[0] ) != null;
if( isDefaultConverter && hasDefaultConstructor )
{
if( options.IgnoreReferences )
{
continue;
}
// If the type is not one covered by our type converters
// and it has a parameterless constructor, create a
// reference map for it.
if( CheckForCircularReference( member.MemberType(), mapParents ) )
{
continue;
}
mapParents.AddLast( type );
var refMapType = typeof( DefaultCsvClassMap<> ).MakeGenericType( member.MemberType() );
var refMap = (CsvClassMap)ReflectionHelper.CreateInstance( refMapType );
var refOptions = options.Copy();
refOptions.IgnoreReferences = false;
AutoMapInternal( refMap, options, mapParents, map.GetMaxIndex() + 1 );
if( refMap.PropertyMaps.Count > 0 || refMap.ReferenceMaps.Count > 0 )
{
var referenceMap = new CsvPropertyReferenceMap( member, refMap );
if( options.PrefixReferenceHeaders )
{
referenceMap.Prefix();
}
map.ReferenceMaps.Add( referenceMap );
}
}
else
{
var propertyMap = new CsvPropertyMap( member );
// Use global values as the starting point.
propertyMap.Data.TypeConverterOptions = TypeConverterOptions.Merge( options.TypeConverterOptionsFactory.GetOptions( member.MemberType() ) );
propertyMap.Data.Index = map.GetMaxIndex() + 1;
if( !isDefaultConverter )
{
// Only add the property/field map if it can be converted later on.
// If the property/field will use the default converter, don't add it because
// we don't want the .ToString() value to be used when auto mapping.
map.PropertyMaps.Add( propertyMap );
}
}
}
map.ReIndex( indexStart );
}
/// <summary>
/// Creates a property expression for the given property on the record.
/// This will recursively traverse the mapping to find the property
/// and create a safe property accessor for each level as it goes.
/// </summary>
/// <param name="recordExpression">The current property expression.</param>
/// <param name="mapping">The mapping to look for the property to map on.</param>
/// <param name="propertyMap">The property map to look for on the mapping.</param>
/// <returns>An Expression to access the given property.</returns>
protected virtual Expression CreatePropertyExpression(Expression recordExpression, CsvClassMap mapping,
CsvPropertyMap propertyMap)
{
Type propertyType;
Expression expression;
Expression expression1;
if (mapping.PropertyMaps.Any(pm => pm == propertyMap))
{
return(Expression.Property(recordExpression, propertyMap.Data.Property));
}
var enumerator = mapping.ReferenceMaps.GetEnumerator();
try
{
while (enumerator.MoveNext())
{
var current = enumerator.Current;
var memberExpression = Expression.Property(recordExpression, current.Data.Property);
var expression2 = CreatePropertyExpression(memberExpression, current.Data.Mapping, propertyMap);
if (expression2 == null)
{
continue;
}
if (!current.Data.Property.PropertyType.GetTypeInfo().IsValueType)
{
var binaryExpression = Expression.Equal(memberExpression, Expression.Constant(null));
bool isValueType = propertyMap.Data.Property.PropertyType.GetTypeInfo().IsValueType;
bool flag = !isValueType
? false
: propertyMap.Data.Property.PropertyType.GetTypeInfo().IsGenericType;
if (!isValueType || flag || configuration.UseNewObjectForNullReferenceProperties)
{
propertyType = propertyMap.Data.Property.PropertyType;
}
else
{
propertyType = typeof(Nullable <>).MakeGenericType(propertyMap.Data.Property.PropertyType);
expression2 = Expression.Convert(expression2, propertyType);
}
if (!isValueType || flag)
{
expression1 = Expression.Constant(null, propertyType);
}
else
{
expression1 = Expression.New(propertyType);
}
expression = Expression.Condition(binaryExpression, expression1, expression2);
return(expression);
}
else
{
expression = expression2;
return(expression);
}
}
return(null);
}
finally
{
((IDisposable)enumerator).Dispose();
}
return(expression);
}
/// <summary>
/// Creates a new instance using the given <see cref="CsvPropertyMap"/>.
/// </summary>
/// <param name="propertyMap">The property/field map the options are being applied to.</param>
public MapTypeConverterOption( CsvPropertyMap propertyMap )
{
this.propertyMap = propertyMap;
}
public ProductMap(string[] allColumns, webModel.CsvImportConfiguration importConfiguration, coreModel.Catalog catalog)
{
var defaultLanguge = catalog.DefaultLanguage != null ? catalog.DefaultLanguage.LanguageCode : "EN-US";
//Dynamical map scalar product fields use by manual mapping information
foreach (var mappingConfigurationItem in importConfiguration.MappingItems.Where(x => x.CsvColumnName != null || x.CustomValue != null))
{
var propertyInfo = typeof(coreModel.CatalogProduct).GetProperty(mappingConfigurationItem.EntityColumnName);
if (propertyInfo != null)
{
var newMap = new CsvPropertyMap(propertyInfo);
newMap.TypeConverterOption(CultureInfo.InvariantCulture);
newMap.TypeConverterOption(NumberStyles.Any);
newMap.TypeConverterOption(true, "yes", "false");
//Map fields if mapping specified
if (mappingConfigurationItem.CsvColumnName != null)
{
newMap.Name(mappingConfigurationItem.CsvColumnName);
}
//And default values if it specified
if (mappingConfigurationItem.CustomValue != null)
{
newMap.Default(mappingConfigurationItem.CustomValue);
}
PropertyMaps.Add(newMap);
}
}
//Map Sku -> Code
Map(x => x.Code).ConvertUsing(x =>
{
return(GetCsvField("Sku", x, importConfiguration));
});
//Map ParentSku -> main product
Map(x => x.MainProductId).ConvertUsing(x =>
{
var parentSku = GetCsvField("ParentSku", x, importConfiguration);
return(!String.IsNullOrEmpty(parentSku) ? parentSku : null);
});
//Map assets (images)
Map(x => x.Assets).ConvertUsing(x =>
{
var retVal = new List <coreModel.ItemAsset>();
var primaryImageUrl = GetCsvField("PrimaryImage", x, importConfiguration);
var altImageUrl = GetCsvField("AltImage", x, importConfiguration);
if (!String.IsNullOrEmpty(primaryImageUrl))
{
retVal.Add(new coreModel.ItemAsset
{
Type = coreModel.ItemAssetType.Image,
Group = "primaryimage",
Url = primaryImageUrl
});
}
if (!String.IsNullOrEmpty(altImageUrl))
{
retVal.Add(new coreModel.ItemAsset
{
Type = coreModel.ItemAssetType.Image,
Group = "image",
Url = altImageUrl
});
}
return(retVal);
});
//Map category
Map(x => x.Category).ConvertUsing(x => new coreModel.Category {
Path = GetCsvField("Category", x, importConfiguration)
});
//Map Reviews
Map(x => x.Reviews).ConvertUsing(x =>
{
var reviews = new List <coreModel.EditorialReview>();
var content = GetCsvField("Review", x, importConfiguration);
if (!String.IsNullOrEmpty(content))
{
reviews.Add(new coreModel.EditorialReview {
Content = content, LanguageCode = defaultLanguge
});
}
return(reviews);
});
//Map Seo
Map(x => x.SeoInfos).ConvertUsing(x =>
{
var seoInfos = new List <SeoInfo>();
var seoUrl = GetCsvField("SeoUrl", x, importConfiguration);
var seoDescription = GetCsvField("SeoDescription", x, importConfiguration);
var seoTitle = GetCsvField("SeoTitle", x, importConfiguration);
if (!String.IsNullOrEmpty(seoUrl) || !String.IsNullOrEmpty(seoTitle) || !String.IsNullOrEmpty(seoDescription))
{
seoUrl = new string[] { seoUrl, seoDescription, seoTitle }.Where(y => !String.IsNullOrEmpty(y)).FirstOrDefault();
seoUrl = seoUrl.Substring(0, Math.Min(seoUrl.Length, 240));
seoInfos.Add(new SeoInfo {
LanguageCode = defaultLanguge, SemanticUrl = seoUrl.GenerateSlug(), MetaDescription = seoDescription, PageTitle = seoTitle
});
}
return(seoInfos);
});
//Map Prices
Map(x => x.Prices).ConvertUsing(x =>
{
var prices = new List <Price>();
var priceId = GetCsvField("PriceId", x, importConfiguration);
var listPrice = GetCsvField("Price", x, importConfiguration);
var salePrice = GetCsvField("SalePrice", x, importConfiguration);
var currency = GetCsvField("Currency", x, importConfiguration);
if (!String.IsNullOrEmpty(listPrice) && !String.IsNullOrEmpty(currency))
{
prices.Add(new Price
{
Id = priceId,
List = Convert.ToDecimal(listPrice, CultureInfo.InvariantCulture),
Sale = salePrice != null ? (decimal?)Convert.ToDecimal(listPrice, CultureInfo.InvariantCulture) : null,
Currency = EnumUtility.SafeParse <CurrencyCodes>(currency, CurrencyCodes.USD)
});
}
return(prices);
});
//Map inventories
Map(x => x.Inventories).ConvertUsing(x =>
{
var inventories = new List <InventoryInfo>();
var quantity = GetCsvField("Quantity", x, importConfiguration);
var allowBackorder = GetCsvField("AllowBackorder", x, importConfiguration);
var fulfilmentCenterId = GetCsvField("FulfilmentCenterId", x, importConfiguration);
if (!String.IsNullOrEmpty(quantity))
{
inventories.Add(new InventoryInfo
{
FulfillmentCenterId = fulfilmentCenterId,
AllowBackorder = allowBackorder.TryParse(false),
InStockQuantity = (long)quantity.TryParse(0.0m)
});
}
return(inventories);
});
//Map properties
if (importConfiguration.PropertyCsvColumns != null && importConfiguration.PropertyCsvColumns.Any())
{
Map(x => x.PropertyValues).ConvertUsing(x => importConfiguration.PropertyCsvColumns.Select(column => new coreModel.PropertyValue {
PropertyName = column, Value = x.GetField <string>(column)
}).ToList());
}
}
public static CsvPropertyMap Field(this CsvPropertyMap source, string name)
{
return(source.Name(name));
}
public async Task WriteRecords <T>(IS3Path path, IEnumerable <T> records)
{
_logger.Information("Writing {Type} records to {@S3Path}", typeof(T).Name, path);
var configuration = new CsvConfiguration
{
Delimiter = "|",
QuoteNoFields = true
};
var customerMap = new DefaultCsvClassMap <T>();
foreach (var prop in typeof(T).GetProperties())
{
var name = prop.Name.ToSnakeCase();
var map = new CsvPropertyMap(prop).Name(name);
customerMap.PropertyMaps.Add(map);
}
configuration.RegisterClassMap(customerMap);
var recordCount = 0;
using (var temp = new TempFile(".md-ra-poc.csv"))
{
_logger.Debug("Writing records to temp path {TempPath}", temp.FullPath);
using (var textWriter = File.CreateText(temp.FullPath))
{
using (var csvWriter = new CsvWriter(textWriter, configuration))
{
// Write header manually because WriteHeader append an index
foreach (var property in csvWriter.Configuration.Maps[typeof(T)].PropertyMaps)
{
csvWriter.WriteField(property.Data.Names.FirstOrDefault());
}
// Advance past header
csvWriter.NextRecord();
foreach (var record in records)
{
csvWriter.WriteRecord(record);
csvWriter.NextRecord();
recordCount++;
}
}
}
_logger.Debug("Wrote CSV to {HierarchyPath}. Records: {Records}. Size: {Bytes} bytes", temp.FullPath, recordCount, new FileInfo(temp.FullPath).Length);
_logger.Debug("Uploading {TempPath} to S3 HierarchyPath: {@S3Path}", temp.FullPath, path);
var transferUtility = new TransferUtility(_s3Client);
await transferUtility.UploadAsync(new TransferUtilityUploadRequest
{
BucketName = path.BucketName,
Key = path.Key,
FilePath = temp.FullPath
}, CancellationToken.None);
_logger.Debug("Uploaded {TempPath} to S3 HierarchyPath: {@S3Path}", temp.FullPath, path);
}
_logger.Information("Uploaded {Records} {Type} records to {@S3Path}", recordCount, typeof(T).Name, path);
}
public DataImportClient()
{
fields.InsertRange(
fields.IndexOf(
fields.Last(
f => f.StartsWith("L.")
)
) + 1, Enumerable.Range(1, 31).Select(
n => "L.UserDef" + n.ToString()
)
);
fields.RemoveRange(fields.IndexOf("L.UserDef14"), 10);
fields.Remove("L.UserDef4");
fields.Remove("L.UserDef6");
fields.Remove("L.UserDef7");
fields.Remove("L.UserDef9");
recordDict = new List <Dictionary <string, string> >();
/*
* groups = fields.GroupBy(
* f => f.Substring(
* 0, f.IndexOf('.')
* ), f => f.Remove(
* 0, f.IndexOf('.') + 1
* )
* ).Select(g => new Group(
* g.Key, g.ToArray()
* )).OrderBy(g => g.prefix).ToArray();*/
cfg = new CsvConfiguration();
cfg.Delimiter = "\t";
cfg.DetectColumnCountChanges = true;
cfg.TrimFields = true;
//cfg.RegisterClassMap(typeof(Group));
var grps = fields.GroupBy(f => f.Substring(0, f.IndexOf('.')));
csvMapping = (
Lookup <string, Lookup <string, string> >
)grps.ToLookup(
f => f.Key, f => (
Lookup <string, string>
)f.ToLookup(
g => g.Remove(
0, g.IndexOf('.') + 1)));
/* var csvMap = new DefaultCsvClassMap<Lookup<string, string>>();
* foreach(var group in csvMapping)
* {
* foreach(var subgroup in group)
* {
* foreach(var item in subgroup)
* {
* var newMap = new CsvPropertyMap(
* typeof(IGrouping<string, string>).GetType().GetProperty(item.Key)
* );
* }
* }
* }*/
fileFormatMap = new DefaultCsvClassMap <FileFormatSkeleton>();
Type ffs = typeof(FileFormatSkeleton);
ffsFields = ffs.GetRuntimeFields() /*ffs.GetFields(BindingFlags.SetProperty)*/.ToDictionary(f => f.Name, f => f);
var fieldNames = fields.Select(f => new KeyValuePair <string, string>(f, f.Replace('.', '_')));
foreach (var field in fieldNames)
{
try
{
var propInfo = ffs.GetProperty(field.Value);
var newMap = new CsvPropertyMap(propInfo);
newMap.Name(field.Key);
fileFormatMap.PropertyMaps.Add(newMap);
}
catch
{
break;
}
}
//cfg.RegisterClassMap(fileFormatMap);
/*FileStream fs = new FileStream(
* @"C:\DocUploads\logs\Export.txt",
* FileMode.Create,
* FileAccess.ReadWrite,
* FileShare.Read
* );
* StreamWriter sw = new StreamWriter(fs);
* CsvWriter writer = new CsvWriter(sw, cfg);*/
}
public static CsvPropertyMap Value <T>(this CsvPropertyMap source, T value)
{
return(source.ConvertUsing(row => value));
}
