private void DiscoverRecordFields(Type recordType)
{
if (!IsDynamicObject) //recordType != typeof(ExpandoObject))
{
CSVRecordFieldConfigurations.Clear();
if (ChoTypeDescriptor.GetProperties(recordType).Where(pd => pd.Attributes.OfType <ChoCSVRecordFieldAttribute>().Any()).Any())
{
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType).Where(pd => pd.Attributes.OfType <ChoCSVRecordFieldAttribute>().Any()))
{
//if (!pd.PropertyType.IsSimple())
// throw new ChoRecordConfigurationException("Property '{0}' is not a simple type.".FormatString(pd.Name));
var obj = new ChoCSVRecordFieldConfiguration(pd.Name, pd.Attributes.OfType <ChoCSVRecordFieldAttribute>().First());
obj.FieldType = pd.PropertyType;
CSVRecordFieldConfigurations.Add(obj);
}
}
else
{
int position = 0;
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType))
{
//if (!pd.PropertyType.IsSimple())
// throw new ChoRecordConfigurationException("Property '{0}' is not a simple type.".FormatString(pd.Name));
var obj = new ChoCSVRecordFieldConfiguration(pd.Name, ++position);
obj.FieldType = pd.PropertyType;
CSVRecordFieldConfigurations.Add(obj);
}
}
}
}
public ChoCSVWriter <T> Index <TField>(Expression <Func <T, TField> > field, int minumum, int maximum)
{
Type recordType = field.GetPropertyType().GetUnderlyingType();
var fqn = field.GetFullyQualifiedMemberName();
if (typeof(IList).IsAssignableFrom(recordType) && !typeof(ArrayList).IsAssignableFrom(recordType) &&
minumum >= 0 && maximum >= 0 && minumum <= maximum)
{
recordType = recordType.GetItemType().GetUnderlyingType();
if (recordType.IsSimple())
{
}
else
{
//Remove any unused config
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType))
{
var fcs = Configuration.CSVRecordFieldConfigurations.Where(o => o.DeclaringMember == "{0}.{1}".FormatString(field.GetFullyQualifiedMemberName(), pd.Name) &&
o.ArrayIndex != null && (o.ArrayIndex <minumum || o.ArrayIndex> maximum)).ToArray();
foreach (var fc in fcs)
{
Configuration.CSVRecordFieldConfigurations.Remove(fc);
}
}
for (int index = minumum; index <= maximum; index++)
{
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType))
{
var fc = Configuration.CSVRecordFieldConfigurations.Where(o => o.DeclaringMember == "{0}.{1}".FormatString(field.GetFullyQualifiedMemberName(), pd.Name) &&
o.ArrayIndex != null && o.ArrayIndex == index).FirstOrDefault();
if (fc != null)
{
continue;
}
Type pt = pd.PropertyType.GetUnderlyingType();
if (pt != typeof(object) && !pt.IsSimple())
{
}
else
{
int fieldPosition = 0;
fieldPosition = Configuration.CSVRecordFieldConfigurations.Count > 0 ? Configuration.CSVRecordFieldConfigurations.Max(f => f.FieldPosition) : 0;
fieldPosition++;
ChoCSVRecordFieldConfiguration obj = Configuration.NewFieldConfiguration(ref fieldPosition, field.GetFullyQualifiedMemberName(), pd, index, field.GetPropertyDescriptor().GetDisplayName());
//if (!CSVRecordFieldConfigurations.Any(c => c.Name == (declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name))))
Configuration.CSVRecordFieldConfigurations.Add(obj);
}
}
}
}
}
return(this);
}
private void DiscoverRecordFields(Type recordType)
{
RecordFieldConfigurations.Clear();
foreach (PropertyDescriptor pd in TypeDescriptor.GetProperties(recordType).AsTypedEnumerable <PropertyDescriptor>().Where(pd => pd.Attributes.OfType <ChoCSVRecordFieldAttribute>().Any()))
{
var obj = new ChoCSVRecordFieldConfiguration(pd.Name, pd.Attributes.OfType <ChoCSVRecordFieldAttribute>().First());
obj.FieldType = pd.PropertyType;
RecordFieldConfigurations.Add(obj);
}
}
private string CleanFieldValue(ChoCSVRecordFieldConfiguration config, string fieldValue)
{
if (fieldValue.IsNull())
{
return(fieldValue);
}
if (fieldValue != null)
{
switch (config.FieldValueTrimOption)
{
case ChoFieldValueTrimOption.Trim:
fieldValue = fieldValue.Trim();
break;
case ChoFieldValueTrimOption.TrimStart:
fieldValue = fieldValue.TrimStart();
break;
case ChoFieldValueTrimOption.TrimEnd:
fieldValue = fieldValue.TrimEnd();
break;
}
}
if (config.Size != null)
{
if (fieldValue.Length > config.Size.Value)
{
if (!config.Truncate)
{
throw new ChoParserException("Incorrect field value length found for '{0}' member [Expected: {1}, Actual: {2}].".FormatString(config.FieldName, config.Size.Value, fieldValue.Length));
}
else
{
fieldValue = fieldValue.Substring(0, config.Size.Value);
}
}
}
if (config.QuoteField != null && config.QuoteField.Value && fieldValue.StartsWith(@"""") && fieldValue.EndsWith(@""""))
{
return(fieldValue.Substring(1, fieldValue.Length - 2));
}
else if ((fieldValue.Contains(Configuration.Delimiter) ||
fieldValue.Contains(Configuration.EOLDelimiter)) && fieldValue.StartsWith(@"""") && fieldValue.EndsWith(@""""))
{
return(fieldValue.Substring(1, fieldValue.Length - 2));
}
else
{
return(fieldValue);
}
}
public ChoCSVReader <T> WithFields(params string[] fieldsNames)
{
string fnTrim = null;
if (!fieldsNames.IsNullOrEmpty())
{
int maxFieldPos = Configuration.CSVRecordFieldConfigurations.Count > 0 ? Configuration.CSVRecordFieldConfigurations.Max(f => f.FieldPosition) : 0;
PropertyDescriptor pd = null;
ChoCSVRecordFieldConfiguration fc = null;
foreach (string fn in fieldsNames)
{
if (fn.IsNullOrEmpty())
{
continue;
}
if (!_clearFields)
{
ClearFields();
Configuration.MapRecordFields(Configuration.RecordType);
//Configuration.ColumnOrderStrict = true;
}
fnTrim = fn.NTrim();
if (Configuration.CSVRecordFieldConfigurations.Any(o => o.Name == fnTrim))
{
fc = Configuration.CSVRecordFieldConfigurations.Where(o => o.Name == fnTrim).First();
Configuration.CSVRecordFieldConfigurations.Remove(Configuration.CSVRecordFieldConfigurations.Where(o => o.Name == fnTrim).First());
}
else
{
pd = ChoTypeDescriptor.GetProperty(typeof(T), fn);
}
var nfc = new ChoCSVRecordFieldConfiguration(fnTrim, ++maxFieldPos)
{
FieldName = fn
};
nfc.PropertyDescriptor = fc != null ? fc.PropertyDescriptor : pd;
nfc.DeclaringMember = fc != null ? fc.DeclaringMember : null;
if (pd != null)
{
if (nfc.FieldType == null)
{
nfc.FieldType = pd.PropertyType;
}
}
Configuration.CSVRecordFieldConfigurations.Add(nfc);
}
}
return(this);
}
public ChoCSVReader <T> DictionaryKeys <TField>(Expression <Func <T, TField> > field, params string[] keys)
{
Type recordType = field.GetPropertyType().GetUnderlyingType();
var fqn = field.GetFullyQualifiedMemberName();
PropertyDescriptor pd = field.GetPropertyDescriptor();
if (recordType.IsGenericType && recordType.GetGenericTypeDefinition() == typeof(Dictionary <,>) &&
typeof(string) == recordType.GetGenericArguments()[0] &&
keys != null && keys.Length > 0)
{
//Remove collection config member
var fcs1 = Configuration.CSVRecordFieldConfigurations.Where(o => o.FieldName == fqn).ToArray();
foreach (var fc in fcs1)
{
Configuration.CSVRecordFieldConfigurations.Remove(fc);
}
//Remove any unused config
var fcs = Configuration.CSVRecordFieldConfigurations.Where(o => o.DeclaringMember == pd.Name &&
!o.DictKey.IsNullOrWhiteSpace() && !keys.Contains(o.DictKey)).ToArray();
foreach (var fc in fcs)
{
Configuration.CSVRecordFieldConfigurations.Remove(fc);
}
foreach (var key in keys)
{
if (!key.IsNullOrWhiteSpace())
{
var fc = Configuration.CSVRecordFieldConfigurations.Where(o => o.DeclaringMember == pd.Name &&
!o.DictKey.IsNullOrWhiteSpace() && key == o.DictKey).FirstOrDefault();
if (fc != null)
{
continue;
}
//ChoCSVRecordFieldConfiguration obj = NewFieldConfiguration(ref position, null, propDesc, dictKey: key);
int fieldPosition = 0;
fieldPosition = Configuration.CSVRecordFieldConfigurations.Count > 0 ? Configuration.CSVRecordFieldConfigurations.Max(f => f.FieldPosition) : 0;
fieldPosition++;
ChoCSVRecordFieldConfiguration obj = Configuration.NewFieldConfiguration(ref fieldPosition, null, field.GetPropertyDescriptor(), dictKey: key);
//if (!CSVRecordFieldConfigurations.Any(c => c.Name == (declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name))))
Configuration.CSVRecordFieldConfigurations.Add(obj);
}
}
}
return(this);
}
internal ChoCSVRecordFieldConfigurationMap(ChoCSVRecordFieldConfiguration config)
{
ChoGuard.ArgumentNotNull(config, nameof(config));
_config = config;
}
private bool ToText(int index, object rec, out string recText)
{
recText = null;
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
ChoCSVRecordFieldConfiguration fieldConfig = null;
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
bool firstColumn = true;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.ThrowAndStopOnMissingField)
{
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
if (!dict.Keys.Contains(kvp.Key, Configuration.FileHeaderConfiguration.StringComparer))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (!ChoType.HasProperty(rec.GetType(), kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (rec is ExpandoObject)
{
IDictionary <string, Object> dict = rec as IDictionary <string, Object>;
fieldValue = dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
}
else
{
if (ChoType.HasProperty(rec.GetType(), kvp.Key))
{
fieldValue = ChoType.GetPropertyValue(rec, kvp.Key);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
fieldValue = rec.GetDefaultValue(kvp.Key, fieldConfig);
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
fieldValue = rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, fieldValue);
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
else if (ChoType.HasProperty(rec.GetType(), kvp.Key) && rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoParserException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
{
fieldText = String.Empty;
}
else
{
fieldText = fieldValue.ToString();
}
if (firstColumn)
{
msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification), GetFillChar(kvp.Value.FillChar), false));
firstColumn = false;
}
else
{
msg.AppendFormat("{0}{1}", Configuration.Delimiter, NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification),
GetFillChar(kvp.Value.FillChar), false));
}
}
recText = msg.ToString();
return(true);
}
internal ChoCSVRecordFieldConfigurationMap(ChoCSVRecordFieldConfiguration config) : base(config)
{
}
private bool ToText(long index, object rec, out string recText)
{
recText = null;
msg.Clear();
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
bool firstColumn = true;
PropertyInfo pi = null;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
if (!dict.ContainsKey(kvp.Key))
{
if (fieldConfig.FieldPosition > dict.Count)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
}
else
{
if (pi == null)
{
pi = Configuration.PIDict.Where(kvp1 => kvp.Value.FieldPosition == kvp.Value.FieldPosition).FirstOrDefault().Value;
}
if (pi == null)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
fieldValue = dict.ContainsKey(kvp.Key) ? dict[kvp.Key] : dict[dict.Keys.ElementAt(fieldConfig.FieldPosition - 1)]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (kvp.Value.FieldType == null)
{
if (fieldValue == null)
{
kvp.Value.FieldType = typeof(string);
}
else
{
kvp.Value.FieldType = fieldValue.GetType();
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
if (fieldConfig.IsDefaultValueSpecified)
{
fieldValue = fieldConfig.DefaultValue;
}
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
}
}
else
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
{
fieldText = String.Empty;
}
else
{
fieldText = fieldValue.ToString();
}
if (firstColumn)
{
msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification), GetFillChar(kvp.Value.FillChar), false));
firstColumn = false;
}
else
{
msg.AppendFormat("{0}{1}", Configuration.Delimiter, NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification),
GetFillChar(kvp.Value.FillChar), false));
}
}
recText = msg.ToString();
return(true);
}
private bool FillRecord(object rec, Tuple <int, string> pair)
{
int lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
object fieldValue = null;
string[] fieldValues = (from x in line.Split(Configuration.Delimiter, Configuration.StringSplitOptions, Configuration.QuoteChar)
select x).ToArray();
if (Configuration.ColumnCountStrict)
{
if (fieldValues.Length != Configuration.CSVRecordFieldConfigurations.Count)
{
throw new ChoParserException("Incorrect number of field values found at line [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.CSVRecordFieldConfigurations.Count, fieldValues.Length, pair.Item1));
}
}
Dictionary <string, string> fieldNameValues = ToFieldNameValues(fieldValues);
ValidateLine(pair.Item1, fieldValues);
ChoCSVRecordFieldConfiguration fieldConfig = null;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.FileHeaderConfiguration.HasHeaderRecord)
{
if (fieldNameValues.ContainsKey(fieldConfig.FieldName))
{
fieldValue = fieldNameValues[fieldConfig.FieldName];
}
else if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field header found.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (fieldConfig.FieldPosition - 1 < fieldValues.Length)
{
fieldValue = fieldValues[fieldConfig.FieldPosition - 1];
}
else if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing field value for {0} [Position: {1}] field.".FormatString(fieldConfig.FieldName, fieldConfig.FieldPosition));
}
}
fieldValue = CleanFieldValue(fieldConfig, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
return(false);
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = null;
}
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture);
if (ignoreFieldValue)
{
dict.AddOrUpdate(kvp.Key, fieldValue);
}
else
{
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
if (ChoType.HasProperty(rec.GetType(), kvp.Key))
{
rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture);
if (!ignoreFieldValue)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (ChoType.HasProperty(rec.GetType(), kvp.Key) && rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoParserException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
private void DiscoverRecordFields(Type recordType, ref int position, string declaringMember = null, bool optIn = false)
{
if (!recordType.IsDynamicType())
{
Type pt = null;
if (optIn) //ChoTypeDescriptor.GetProperties(recordType).Where(pd => pd.Attributes.OfType<ChoCSVRecordFieldAttribute>().Any()).Any())
{
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType))
{
pt = pd.PropertyType.GetUnderlyingType();
if (!pt.IsSimple() && !typeof(IEnumerable).IsAssignableFrom(pt))
{
DiscoverRecordFields(pt, ref position, declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), optIn);
}
else if (pd.Attributes.OfType <ChoCSVRecordFieldAttribute>().Any())
{
var obj = new ChoCSVRecordFieldConfiguration(pd.Name, pd.Attributes.OfType <ChoCSVRecordFieldAttribute>().First(), pd.Attributes.OfType <Attribute>().ToArray());
obj.FieldType = pt;
obj.PropertyDescriptor = pd;
obj.DeclaringMember = declaringMember == null ? null : "{0}.{1}".FormatString(declaringMember, pd.Name);
if (!CSVRecordFieldConfigurations.Any(c => c.Name == pd.Name))
{
CSVRecordFieldConfigurations.Add(obj);
}
}
}
}
else
{
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType))
{
pt = pd.PropertyType.GetUnderlyingType();
if (pt != typeof(object) && !pt.IsSimple() && !typeof(IEnumerable).IsAssignableFrom(pt))
{
DiscoverRecordFields(pt, ref position, declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), optIn);
}
else
{
var obj = new ChoCSVRecordFieldConfiguration(pd.Name, ++position);
obj.FieldType = pd.PropertyType; // pt;
obj.PropertyDescriptor = pd;
obj.DeclaringMember = declaringMember == null ? null : "{0}.{1}".FormatString(declaringMember, pd.Name);
StringLengthAttribute slAttr = pd.Attributes.OfType <StringLengthAttribute>().FirstOrDefault();
if (slAttr != null && slAttr.MaximumLength > 0)
{
obj.Size = slAttr.MaximumLength;
}
DisplayAttribute dpAttr = pd.Attributes.OfType <DisplayAttribute>().FirstOrDefault();
if (dpAttr != null)
{
if (!dpAttr.ShortName.IsNullOrWhiteSpace())
{
obj.FieldName = dpAttr.ShortName;
}
else if (!dpAttr.Name.IsNullOrWhiteSpace())
{
obj.FieldName = dpAttr.Name;
}
}
DisplayFormatAttribute dfAttr = pd.Attributes.OfType <DisplayFormatAttribute>().FirstOrDefault();
if (dfAttr != null && !dfAttr.DataFormatString.IsNullOrWhiteSpace())
{
obj.FormatText = dfAttr.DataFormatString;
}
if (dfAttr != null && !dfAttr.NullDisplayText.IsNullOrWhiteSpace())
{
obj.NullValue = dfAttr.NullDisplayText;
}
if (!CSVRecordFieldConfigurations.Any(c => c.Name == pd.Name))
{
CSVRecordFieldConfigurations.Add(obj);
}
}
}
}
}
}
private bool ToText(long index, object rec, out string recText)
{
if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType))
{
rec = ChoActivator.CreateInstance(Configuration.RecordType, rec);
}
recText = null;
msg.Clear();
if (Configuration.ColumnCountStrict)
{
CheckColumnCountStrict(rec);
}
if (Configuration.ColumnOrderStrict)
{
CheckColumnOrderStrict(rec);
}
bool firstColumn = true;
PropertyInfo pi = null;
object rootRec = rec;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
if (Configuration.IsDynamicObject)
{
if (Configuration.IgnoredFields.Contains(kvp.Key))
{
continue;
}
}
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (Configuration.IsDynamicObject && Configuration.UseNestedKeyFormat)
{
dict = dict.Flatten(Configuration.NestedColumnSeparator).ToDictionary();
}
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
if (!dict.ContainsKey(kvp.Key))
{
if (fieldConfig.FieldPosition > dict.Count)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
}
else
{
if (pi == null)
{
pi = Configuration.PIDict.Where(kvp1 => kvp.Value.FieldPosition == kvp.Value.FieldPosition).FirstOrDefault().Value;
}
if (pi == null)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
fieldValue = dict.ContainsKey(kvp.Key) ? dict[kvp.Key] : dict[dict.Keys.ElementAt(fieldConfig.FieldPosition - 1)]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (rec is ChoDynamicObject)
{
var dobj = rec as ChoDynamicObject;
kvp.Value.FieldType = dobj.GetMemberType(kvp.Key);
}
if (kvp.Value.FieldType == null)
{
if (fieldValue == null)
{
kvp.Value.FieldType = typeof(string);
}
else
{
kvp.Value.FieldType = fieldValue.GetType();
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
if (fieldConfig.IsDefaultValueSpecified)
{
fieldValue = fieldConfig.DefaultValue;
}
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
if (fieldConfig.ValueSelector == null)
{
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else
{
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue);
}
}
else
{
fieldValue = fieldConfig.ValueSelector(rec);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
{
fieldText = String.Empty;
}
else
{
if (fieldValue is IList)
{
StringBuilder sb = new StringBuilder();
bool first = true;
foreach (var item in (IList)fieldValue)
{
if (first)
{
sb.Append(NormalizeFieldValue(kvp.Key, item.ToNString(), null, false, null, ChoFieldValueJustification.None, GetFillChar(kvp.Value.FillChar)));
first = false;
}
else
{
sb.AppendFormat("{0}{1}", Configuration.Delimiter, NormalizeFieldValue(kvp.Key, item.ToNString(), null, false, null, ChoFieldValueJustification.None, GetFillChar(kvp.Value.FillChar)));
}
}
fieldText = sb.ToString();
}
else
{
fieldText = fieldValue.ToString();
}
}
if (firstColumn)
{
msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField,
GetFieldValueJustification(kvp.Value.FieldValueJustification), GetFillChar(kvp.Value.FillChar),
false, kvp.Value.NullValue, kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType, Configuration.FieldValueTrimOption), fieldConfig));
firstColumn = false;
}
else
{
msg.AppendFormat("{0}{1}", Configuration.Delimiter, NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification),
GetFillChar(kvp.Value.FillChar), false, kvp.Value.NullValue, kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType, Configuration.FieldValueTrimOption), fieldConfig));
}
}
recText = msg.ToString();
return(true);
}
internal ChoCSVRecordFieldConfiguration NewFieldConfiguration(ref int position, string declaringMember, PropertyDescriptor pd,
int?arrayIndex = null, string displayName = null, string dictKey = null)
{
ChoCSVRecordFieldConfiguration obj = null;
if (displayName.IsNullOrEmpty())
{
obj = new ChoCSVRecordFieldConfiguration(declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), ++position);
}
else
{
obj = new ChoCSVRecordFieldConfiguration("{0}.{1}".FormatString(displayName, pd.Name), ++position);
}
obj.DictKey = dictKey;
obj.ArrayIndex = arrayIndex;
obj.FieldType = pd.PropertyType; // pt;
obj.PropertyDescriptor = pd;
obj.DeclaringMember = declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name);
if (arrayIndex == null)
{
ChoFieldPositionAttribute fpAttr = pd.Attributes.OfType <ChoFieldPositionAttribute>().FirstOrDefault();
if (fpAttr != null && fpAttr.Position > 0)
{
obj.FieldPosition = fpAttr.Position;
}
}
else
{
}
StringLengthAttribute slAttr = pd.Attributes.OfType <StringLengthAttribute>().FirstOrDefault();
if (slAttr != null && slAttr.MaximumLength > 0)
{
obj.Size = slAttr.MaximumLength;
}
DisplayNameAttribute dnAttr = pd.Attributes.OfType <DisplayNameAttribute>().FirstOrDefault();
if (dnAttr != null && !dnAttr.DisplayName.IsNullOrWhiteSpace())
{
obj.FieldName = dnAttr.DisplayName.Trim();
}
else
{
DisplayAttribute dpAttr = pd.Attributes.OfType <DisplayAttribute>().FirstOrDefault();
if (dpAttr != null)
{
if (!dpAttr.ShortName.IsNullOrWhiteSpace())
{
obj.FieldName = dpAttr.ShortName.Trim();
}
else if (!dpAttr.Name.IsNullOrWhiteSpace())
{
obj.FieldName = dpAttr.Name.Trim();
}
}
}
DisplayFormatAttribute dfAttr = pd.Attributes.OfType <DisplayFormatAttribute>().FirstOrDefault();
if (dfAttr != null && !dfAttr.DataFormatString.IsNullOrWhiteSpace())
{
obj.FormatText = dfAttr.DataFormatString;
}
if (dfAttr != null && !dfAttr.NullDisplayText.IsNullOrWhiteSpace())
{
obj.NullValue = dfAttr.NullDisplayText;
}
if (arrayIndex != null)
{
if (ArrayIndexSeparator == null)
{
obj.FieldName = obj.FieldName + "_" + arrayIndex;
}
else
{
obj.FieldName = obj.FieldName + ArrayIndexSeparator + arrayIndex;
}
}
else if (!dictKey.IsNullOrWhiteSpace())
{
obj.FieldName = dictKey;
}
return(obj);
}
private void DiscoverRecordFields(Type recordType, ref int position, string declaringMember = null,
bool optIn = false, PropertyDescriptor propDesc = null)
{
if (!recordType.IsDynamicType())
{
Type pt = null;
if (optIn) //ChoTypeDescriptor.GetProperties(recordType).Where(pd => pd.Attributes.OfType<ChoCSVRecordFieldAttribute>().Any()).Any())
{
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType))
{
pt = pd.PropertyType.GetUnderlyingType();
if (!pt.IsSimple() && !typeof(IEnumerable).IsAssignableFrom(pt))
{
DiscoverRecordFields(pt, ref position, declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), optIn);
}
else if (pd.Attributes.OfType <ChoCSVRecordFieldAttribute>().Any())
{
var obj = new ChoCSVRecordFieldConfiguration(pd.Name, pd.Attributes.OfType <ChoCSVRecordFieldAttribute>().First(), pd.Attributes.OfType <Attribute>().ToArray());
obj.FieldType = pt;
obj.PropertyDescriptor = pd;
obj.DeclaringMember = declaringMember == null ? null : "{0}.{1}".FormatString(declaringMember, pd.Name);
if (!CSVRecordFieldConfigurations.Any(c => c.Name == pd.Name))
{
CSVRecordFieldConfigurations.Add(obj);
}
}
}
}
else
{
if (typeof(IList).IsAssignableFrom(recordType) &&
!typeof(ArrayList).IsAssignableFrom(recordType) &&
!recordType.IsInterface)
{
if (propDesc != null)
{
RangeAttribute dnAttr = propDesc.Attributes.OfType <RangeAttribute>().FirstOrDefault();
if (dnAttr == null)
{
ChoCSVRecordFieldConfiguration obj = NewFieldConfiguration(ref position, null, propDesc, null, declaringMember == null ? propDesc.GetDisplayName() : propDesc.GetDisplayName(String.Empty));
CSVRecordFieldConfigurations.Add(obj);
}
else if (dnAttr != null && dnAttr.Minimum.CastTo <int>() >= 0 && dnAttr.Maximum.CastTo <int>() > 0 &&
dnAttr.Minimum.CastTo <int>() <= dnAttr.Maximum.CastTo <int>())
{
recordType = recordType.GetItemType().GetUnderlyingType();
if (recordType.IsSimple())
{
for (int range = dnAttr.Minimum.CastTo <int>(); range <= dnAttr.Maximum.CastTo <int>(); range++)
{
ChoCSVRecordFieldConfiguration obj = NewFieldConfiguration(ref position, null, propDesc, range);
//if (!CSVRecordFieldConfigurations.Any(c => c.Name == (declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name))))
CSVRecordFieldConfigurations.Add(obj);
}
}
else
{
for (int range = dnAttr.Minimum.CastTo <int>(); range <= dnAttr.Maximum.CastTo <int>(); range++)
{
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType))
{
pt = pd.PropertyType.GetUnderlyingType();
if (pt != typeof(object) && !pt.IsSimple() /*&& !typeof(IEnumerable).IsAssignableFrom(pt)*/)
{
//DiscoverRecordFields(pt, ref position, declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), optIn, pd);
}
else
{
ChoCSVRecordFieldConfiguration obj = NewFieldConfiguration(ref position, declaringMember, pd, range, propDesc.GetDisplayName());
//if (!CSVRecordFieldConfigurations.Any(c => c.Name == (declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name))))
CSVRecordFieldConfigurations.Add(obj);
}
}
}
}
}
}
}
else if (recordType.IsGenericType && recordType.GetGenericTypeDefinition() == typeof(Dictionary <,>)
/*&& typeof(string) == recordType.GetGenericArguments()[0]*/)
{
if (propDesc != null)
{
ChoDictionaryKeyAttribute[] dnAttrs = propDesc.Attributes.OfType <ChoDictionaryKeyAttribute>().ToArray();
if (dnAttrs.IsNullOrEmpty())
{
ChoCSVRecordFieldConfiguration obj = NewFieldConfiguration(ref position, null, propDesc, null, declaringMember == null ? propDesc.GetDisplayName() : propDesc.GetDisplayName(String.Empty));
CSVRecordFieldConfigurations.Add(obj);
}
else
{
var keys = (from a in dnAttrs
where a != null && !a.Keys.IsNullOrWhiteSpace()
select a.Keys.SplitNTrim()).SelectMany(a => a).ToArray();
foreach (var key in keys)
{
if (!key.IsNullOrWhiteSpace())
{
ChoCSVRecordFieldConfiguration obj = NewFieldConfiguration(ref position, null, propDesc, dictKey: key);
//if (!CSVRecordFieldConfigurations.Any(c => c.Name == (declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name))))
CSVRecordFieldConfigurations.Add(obj);
}
}
}
}
}
else
{
if (recordType == typeof(object)
//|| typeof(IEnumerable).IsAssignableFrom(recordType)
//|| typeof(ICollection).IsAssignableFrom(recordType)
)
{
}
else if (recordType.IsSimple())
{
ChoCSVRecordFieldConfiguration obj = NewFieldConfiguration(ref position, declaringMember, propDesc);
if (!CSVRecordFieldConfigurations.Any(c => c.Name == propDesc.Name))
{
CSVRecordFieldConfigurations.Add(obj);
}
}
else
{
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType))
{
pt = pd.PropertyType.GetUnderlyingType();
if (pt == typeof(object) ||
typeof(ArrayList).IsAssignableFrom(pt) ||
typeof(Hashtable).IsAssignableFrom(pt)
)
{
continue;
}
if (pt != typeof(object) && !pt.IsSimple() /*&& !typeof(IEnumerable).IsAssignableFrom(pt)*/)
{
if (propDesc == null)
{
DiscoverRecordFields(pt, ref position, declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), optIn, pd);
}
else
{
DiscoverRecordFields(pt, ref position, declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), optIn, pd);
}
}
else
{
ChoCSVRecordFieldConfiguration obj = NewFieldConfiguration(ref position, declaringMember, pd, null, declaringMember == null ? propDesc.GetDisplayName() : propDesc.GetDisplayName(String.Empty));
if (!CSVRecordFieldConfigurations.Any(c => c.Name == (declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name))))
{
CSVRecordFieldConfigurations.Add(obj);
}
}
}
}
}
}
}
}
private string CleanFieldValue(ChoCSVRecordFieldConfiguration config, Type fieldType, string fieldValue)
{
if (fieldValue == null)
{
return(fieldValue);
}
ChoFieldValueTrimOption fieldValueTrimOption = ChoFieldValueTrimOption.Trim;
if (config.FieldValueTrimOption == null)
{
//if (fieldType == typeof(string))
// fieldValueTrimOption = ChoFieldValueTrimOption.None;
}
else
{
fieldValueTrimOption = config.FieldValueTrimOption.Value;
}
switch (fieldValueTrimOption)
{
case ChoFieldValueTrimOption.Trim:
fieldValue = fieldValue.Trim();
break;
case ChoFieldValueTrimOption.TrimStart:
fieldValue = fieldValue.TrimStart();
break;
case ChoFieldValueTrimOption.TrimEnd:
fieldValue = fieldValue.TrimEnd();
break;
}
if (config.Size != null)
{
if (fieldValue.Length > config.Size.Value)
{
if (!config.Truncate)
{
throw new ChoParserException("Incorrect field value length found for '{0}' member [Expected: {1}, Actual: {2}].".FormatString(config.FieldName, config.Size.Value, fieldValue.Length));
}
else
{
fieldValue = fieldValue.Substring(0, config.Size.Value);
}
}
}
char startChar;
char endChar;
//quotes are quoted and doubled (excel) i.e. 15" -> field1,"15""",field3
if (fieldValue.Contains(Configuration.DoubleQuoteChar))
{
fieldValue = fieldValue.Replace(Configuration.DoubleQuoteChar, Configuration.QuoteChar.ToString());
}
if (fieldValue.Contains(Configuration.BackslashQuote))
{
fieldValue = fieldValue.Replace(Configuration.BackslashQuote, Configuration.QuoteChar.ToString());
}
if (fieldValue.Length >= 2)
{
startChar = fieldValue[0];
endChar = fieldValue[fieldValue.Length - 1];
if (config.QuoteField != null && config.QuoteField.Value && startChar == Configuration.QuoteChar && endChar == Configuration.QuoteChar)
{
return(fieldValue.Substring(1, fieldValue.Length - 2));
}
else if (startChar == Configuration.QuoteChar && endChar == Configuration.QuoteChar &&
(fieldValue.Contains(Configuration.Delimiter) ||
fieldValue.Contains(Configuration.EOLDelimiter)))
{
return(fieldValue.Substring(1, fieldValue.Length - 2));
}
}
return(fieldValue);
}
private bool FillRecord(object rec, Tuple <long, string> pair)
{
long lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
string[] fieldValues = line.Split(Configuration.Delimiter, Configuration.StringSplitOptions, Configuration.QuoteChar);
if (Configuration.ColumnCountStrict)
{
if (fieldValues.Length != Configuration.CSVRecordFieldConfigurations.Count)
{
throw new ChoParserException("Incorrect number of field values found at line [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.CSVRecordFieldConfigurations.Count, fieldValues.Length, pair.Item1));
}
}
if (_fieldNames != null) //Configuration.FileHeaderConfiguration.HasHeaderRecord && Configuration.ColumnOrderStrict)
{
if (this.fieldNameValues == null)
{
this.fieldNameValues = InitFieldNameValuesDict();
}
ToFieldNameValues(fieldNameValues, fieldValues);
}
ValidateLine(pair.Item1, fieldValues);
object fieldValue = null;
ChoCSVRecordFieldConfiguration fieldConfig = null;
PropertyInfo pi = null;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.FCArray)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
if (fieldNameValues != null)
{
if (fieldNameValues.ContainsKey(fieldConfig.FieldName))
{
fieldValue = fieldNameValues[fieldConfig.FieldName];
}
else
{
if (Configuration.ColumnOrderStrict)
{
throw new ChoParserException("No matching '{0}' field header found.".FormatString(fieldConfig.FieldName));
}
}
}
else
{
if (fieldConfig.FieldPosition - 1 < fieldValues.Length)
{
fieldValue = fieldValues[fieldConfig.FieldPosition - 1];
}
else
{
throw new ChoParserException("Missing field value for '{0}' [Position: {1}] field.".FormatString(fieldConfig.FieldName, fieldConfig.FieldPosition));
}
}
//if (Configuration.FileHeaderConfiguration.HasHeaderRecord && Configuration.ColumnOrderStrict)
//{
// if (fieldNameValues.ContainsKey(fieldConfig.FieldName))
// fieldValue = fieldNameValues[fieldConfig.FieldName];
// else if (Configuration.ColumnCountStrict)
// throw new ChoParserException("No matching '{0}' field header found.".FormatString(fieldConfig.FieldName));
//}
//else
//{
// if (fieldConfig.FieldPosition - 1 < fieldValues.Length)
// fieldValue = fieldValues[fieldConfig.FieldPosition - 1];
// else if (Configuration.ColumnCountStrict)
// throw new ChoParserException("Missing field value for '{0}' [Position: {1}] field.".FormatString(fieldConfig.FieldName, fieldConfig.FieldPosition));
//}
if (Configuration.IsDynamicObject)
{
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = typeof(string);
}
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
internal ChoCSVRecordFieldConfigurationMap(ChoCSVRecordFieldConfiguration config)
{
_config = config;
}
private ChoCSVReader <T> WithField(string name, int?position, Type fieldType = null, bool?quoteField = null,
ChoFieldValueTrimOption fieldValueTrimOption = ChoFieldValueTrimOption.Trim, string fieldName = null,
Func <object, object> valueConverter = null,
Func <dynamic, object> valueSelector = null,
object defaultValue = null, object fallbackValue = null, string altFieldNames = null,
string fullyQualifiedMemberName = null, string formatText = null,
string nullValue = null)
{
if (!name.IsNullOrEmpty())
{
if (!_clearFields)
{
ClearFields();
Configuration.MapRecordFields(Configuration.RecordType);
}
if (fieldName.IsNullOrWhiteSpace())
{
fieldName = name;
}
string fnTrim = name.NTrim();
ChoCSVRecordFieldConfiguration fc = null;
PropertyDescriptor pd = null;
if (Configuration.CSVRecordFieldConfigurations.Any(o => o.Name == fnTrim))
{
fc = Configuration.CSVRecordFieldConfigurations.Where(o => o.Name == fnTrim).First();
if (position == null)
{
position = fc.FieldPosition;
}
Configuration.CSVRecordFieldConfigurations.Remove(fc);
}
else
{
pd = ChoTypeDescriptor.GetNestedProperty(typeof(T), fullyQualifiedMemberName.IsNullOrWhiteSpace() ? name : fullyQualifiedMemberName);
position = Configuration.CSVRecordFieldConfigurations.Count > 0 ? Configuration.CSVRecordFieldConfigurations.Max(f => f.FieldPosition) : 0;
position++;
}
var nfc = new ChoCSVRecordFieldConfiguration(fnTrim, position.Value)
{
FieldType = fieldType,
QuoteField = quoteField,
FieldValueTrimOption = fieldValueTrimOption,
FieldName = fieldName,
ValueConverter = valueConverter,
ValueSelector = valueSelector,
DefaultValue = defaultValue,
FallbackValue = fallbackValue,
AltFieldNames = altFieldNames,
FormatText = formatText,
NullValue = nullValue
};
if (fullyQualifiedMemberName.IsNullOrWhiteSpace())
{
nfc.PropertyDescriptor = fc != null ? fc.PropertyDescriptor : pd;
nfc.DeclaringMember = fc != null ? fc.DeclaringMember : fullyQualifiedMemberName;
}
else
{
pd = ChoTypeDescriptor.GetNestedProperty(typeof(T), fullyQualifiedMemberName);
nfc.PropertyDescriptor = pd;
nfc.DeclaringMember = fullyQualifiedMemberName;
}
if (pd != null)
{
if (nfc.FieldType == null)
{
nfc.FieldType = pd.PropertyType;
}
}
Configuration.CSVRecordFieldConfigurations.Add(nfc);
}
return(this);
}
private ChoCSVWriter <T> WithField(string name, Type fieldType = null, bool?quoteField = null, char?fillChar = null,
ChoFieldValueJustification?fieldValueJustification = null,
bool?truncate = null, string fieldName = null, int?fieldPosition = null,
Func <object, object> valueConverter = null,
Func <dynamic, object> valueSelector = null,
Func <string> headerSelector = null,
object defaultValue = null, object fallbackValue = null,
string fullyQualifiedMemberName = null, string formatText = null, string nullValue = null,
bool excelField = false)
{
if (!name.IsNullOrEmpty())
{
if (!_clearFields)
{
ClearFields();
Configuration.MapRecordFields(Configuration.RecordType);
}
if (fieldName.IsNullOrWhiteSpace())
{
fieldName = name;
}
string fnTrim = name.NTrim();
ChoCSVRecordFieldConfiguration fc = null;
PropertyDescriptor pd = null;
//if (Configuration.CSVRecordFieldConfigurations.Any(o => o.Name == fnTrim))
//{
if (fullyQualifiedMemberName != null)
{
fc = Configuration.CSVRecordFieldConfigurations.Where(o => o.DeclaringMember == fullyQualifiedMemberName).FirstOrDefault();
}
else
{
fc = Configuration.CSVRecordFieldConfigurations.Where(o => o.Name == fnTrim).FirstOrDefault();
}
if (fc != null && fieldPosition == null)
{
fieldPosition = fc.FieldPosition;
}
//Configuration.CSVRecordFieldConfigurations.Remove(fc);
//}
if (fc == null)
{
pd = ChoTypeDescriptor.GetNestedProperty(typeof(T), fullyQualifiedMemberName.IsNullOrWhiteSpace() ? name : fullyQualifiedMemberName);
if (fieldPosition == null)
{
fieldPosition = Configuration.CSVRecordFieldConfigurations.Count > 0 ? Configuration.CSVRecordFieldConfigurations.Max(f => f.FieldPosition) : 0;
fieldPosition++;
}
}
ChoCSVRecordFieldConfiguration nfc = null;
if (fc != null)
{
nfc = fc;
if (fieldType != null)
{
nfc.FieldType = fieldType;
}
if (quoteField != null)
{
nfc.QuoteField = quoteField;
}
if (fillChar != null)
{
nfc.FillChar = fillChar;
}
if (fieldValueJustification != null)
{
nfc.FieldValueJustification = fieldValueJustification;
}
if (truncate != null)
{
nfc.Truncate = truncate.Value;
}
if (fieldName != null)
{
nfc.FieldName = fieldName;
}
if (valueConverter != null)
{
nfc.ValueConverter = valueConverter;
}
if (valueSelector != null)
{
nfc.ValueSelector = valueSelector;
}
if (headerSelector != null)
{
nfc.HeaderSelector = headerSelector;
}
if (defaultValue != null)
{
nfc.DefaultValue = defaultValue;
}
if (fallbackValue != null)
{
nfc.FallbackValue = fallbackValue;
}
if (formatText != null)
{
nfc.FormatText = formatText;
}
if (nullValue != null)
{
nfc.NullValue = nullValue;
}
nfc.ExcelField = excelField;
}
else
{
nfc = new ChoCSVRecordFieldConfiguration(fnTrim, fieldPosition.Value)
{
FieldType = fieldType,
QuoteField = quoteField,
FillChar = fillChar,
FieldValueJustification = fieldValueJustification,
Truncate = truncate != null ? truncate.Value : true,
FieldName = fieldName,
ValueConverter = valueConverter,
ValueSelector = valueSelector,
HeaderSelector = headerSelector,
DefaultValue = defaultValue,
FallbackValue = fallbackValue,
FormatText = formatText,
NullValue = nullValue,
ExcelField = excelField
};
if (fullyQualifiedMemberName.IsNullOrWhiteSpace())
{
nfc.PropertyDescriptor = fc != null ? fc.PropertyDescriptor : pd;
nfc.DeclaringMember = fc != null ? fc.DeclaringMember : fullyQualifiedMemberName;
}
else
{
pd = ChoTypeDescriptor.GetNestedProperty(typeof(T), fullyQualifiedMemberName);
nfc.PropertyDescriptor = pd;
nfc.DeclaringMember = fullyQualifiedMemberName;
}
if (pd != null)
{
if (nfc.FieldType == null)
{
nfc.FieldType = pd.PropertyType;
}
}
Configuration.CSVRecordFieldConfigurations.Add(nfc);
}
}
return(this);
}
private string NormalizeFieldValue(string fieldName, string fieldValue, int?size, bool truncate, bool?quoteField,
ChoFieldValueJustification fieldValueJustification, char fillChar, bool isHeader = false, string nullValue = null,
ChoFieldValueTrimOption?fieldValueTrimOption = null, ChoCSVRecordFieldConfiguration fieldConfig = null)
{
string lFieldValue = fieldValue;
bool retValue = false;
quoteValue = quoteField != null ? quoteField.Value : false;
if (retValue)
{
return(lFieldValue);
}
if (fieldValue.IsNull())
{
fieldValue = String.Empty;
}
if (fieldValue.StartsWith(Configuration.QuoteChar.ToString()) && fieldValue.EndsWith(Configuration.QuoteChar.ToString()))
{
}
else
{
if (quoteField == null)
{
if (searchStrings == null)
{
searchStrings = (Configuration.QuoteChar.ToString() + Configuration.Delimiter + Configuration.EOLDelimiter).ToArray();
}
if (fieldValue.IndexOfAny(searchStrings) >= 0)
{
//******** ORDER IMPORTANT *********
//Fields that contain double quote characters must be surounded by double-quotes, and the embedded double-quotes must each be represented by a pair of consecutive double quotes.
if (fieldValue.IndexOf(Configuration.QuoteChar) >= 0)
{
if (!Configuration.EscapeQuoteAndDelimiter)
{
fieldValue = fieldValue.Replace(Configuration.QuoteChar.ToString(), Configuration.DoubleQuoteChar);
}
else
{
fieldValue = fieldValue.Replace(Configuration.QuoteChar.ToString(), "\\{0}".FormatString(Configuration.QuoteChar));
}
quoteValue = true;
}
if (fieldValue.IndexOf(Configuration.Delimiter) >= 0)
{
if (isHeader)
{
if (fieldConfig == null || fieldConfig.ValueSelector == null)
{
throw new ChoParserException("Field header '{0}' value contains delimiter character.".FormatString(fieldName));
}
}
else
{
//Fields with embedded commas must be delimited with double-quote characters.
if (Configuration.EscapeQuoteAndDelimiter)
{
fieldValue = fieldValue.Replace(Configuration.Delimiter, "\\{0}".FormatString(Configuration.Delimiter));
}
quoteValue = true;
//throw new ChoParserException("Field '{0}' value contains delimiter character.".FormatString(fieldName));
}
}
if (fieldValue.IndexOf(Configuration.EOLDelimiter) >= 0)
{
if (isHeader)
{
throw new ChoParserException("Field header '{0}' value contains EOL delimiter character.".FormatString(fieldName));
}
else
{
//A field that contains embedded line-breaks must be surounded by double-quotes
//if (Configuration.EscapeQuoteAndDelimiters)
// fieldValue = fieldValue.Replace(Configuration.EOLDelimiter, "\\{0}".FormatString(Configuration.EOLDelimiter));
quoteValue = true;
//throw new ChoParserException("Field '{0}' value contains EOL delimiter character.".FormatString(fieldName));
}
}
}
if (!isHeader)
{
//Fields with leading or trailing spaces must be delimited with double-quote characters.
if (!fieldValue.IsNullOrWhiteSpace() && (char.IsWhiteSpace(fieldValue[0]) || char.IsWhiteSpace(fieldValue[fieldValue.Length - 1])))
{
quoteValue = true;
}
}
}
else
{
quoteValue = quoteField.Value;
}
}
//}
//else
//{
// if (fieldValue.StartsWith(Configuration.QuoteChar.ToString()) && fieldValue.EndsWith(Configuration.QuoteChar.ToString()))
// {
// }
// else
// {
// //Fields that contain double quote characters must be surrounded by double-quotes, and the embedded double-quotes must each be represented by a pair of consecutive double quotes.
// if (fieldValue.IndexOf(Configuration.QuoteChar) >= 0)
// {
// fieldValue = "{1}{0}{1}".FormatString(fieldValue.Replace(Configuration.QuoteChar.ToString(), Configuration.DoubleQuoteChar), Configuration.QuoteChar);
// }
// else
// fieldValue = "{1}{0}{1}".FormatString(fieldValue, Configuration.QuoteChar);
// }
//}
//if (quoteValue)
// size = size - 2;
if (fieldValue.IsNullOrEmpty())
{
if (nullValue != null)
{
fieldValue = nullValue;
}
}
if (size != null)
{
if (quoteValue)
{
size = size.Value - 2;
}
if (size <= 0)
{
return(String.Empty);
}
if (fieldValue.Length < size.Value)
{
if (fillChar != ChoCharEx.NUL)
{
if (fieldValueJustification == ChoFieldValueJustification.Right)
{
fieldValue = fieldValue.PadLeft(size.Value, fillChar);
}
else if (fieldValueJustification == ChoFieldValueJustification.Left)
{
fieldValue = fieldValue.PadRight(size.Value, fillChar);
}
}
}
else if (fieldValue.Length > size.Value)
{
if (truncate)
{
if (fieldValueTrimOption != null)
{
if (fieldValueTrimOption == ChoFieldValueTrimOption.TrimStart)
{
fieldValue = fieldValue.Right(size.Value);
}
else
{
fieldValue = fieldValue.Substring(0, size.Value);
}
}
else
{
fieldValue = fieldValue.Substring(0, size.Value);
}
}
else
{
if (isHeader)
{
throw new ApplicationException("Field header value length overflowed for '{0}' member [Expected: {1}, Actual: {2}].".FormatString(fieldName, size, fieldValue.Length));
}
else
{
throw new ApplicationException("Field value length overflowed for '{0}' member [Expected: {1}, Actual: {2}].".FormatString(fieldName, size, fieldValue.Length));
}
}
}
}
if (fieldConfig != null && fieldConfig.ValueSelector != null)
{
quoteValue = false;
}
if (quoteValue)
{
fieldValue = "{1}{0}{1}".FormatString(fieldValue, Configuration.QuoteChar);
}
return(fieldValue);
}
//public ChoCSVReader<T> Index(string fieldName, Type fieldType, int minumum, int maximum)
//{
// return IndexInternal(fieldName, fieldType, minumum, maximum);
//}
private ChoCSVReader <T> IndexInternal(string fieldName, Type fieldType, int minumum, int maximum,
string fullyQualifiedMemberName = null, string displayName = null)
{
if (fieldType == null)
{
return(this);
}
if (fullyQualifiedMemberName.IsNullOrWhiteSpace())
{
fullyQualifiedMemberName = fieldName;
}
//if (displayName.IsNullOrWhiteSpace())
// displayName = fieldName;
Type recordType = fieldType;
var fqn = fieldName;
if (typeof(IList).IsAssignableFrom(recordType) && !typeof(ArrayList).IsAssignableFrom(recordType) &&
minumum >= 0 && maximum >= 0 && minumum <= maximum &&
!fieldType.IsInterface)
{
var itemType = recordType.GetItemType().GetUnderlyingType();
if (itemType.IsSimple())
{
for (int index = minumum; index <= maximum; index++)
{
int fieldPosition = 0;
fieldPosition = Configuration.CSVRecordFieldConfigurations.Count > 0 ? Configuration.CSVRecordFieldConfigurations.Max(f => f.FieldPosition) : 0;
fieldPosition++;
string lFieldName = null;
if (Configuration.ArrayIndexSeparator == null)
{
lFieldName = fieldName + "_" + index;
}
else
{
lFieldName = fieldName + Configuration.ArrayIndexSeparator + index;
}
var nfc = new ChoCSVRecordFieldConfiguration(lFieldName, fieldPosition)
{
ArrayIndex = index
};
nfc.FieldType = recordType;
Configuration.CSVRecordFieldConfigurations.Add(nfc);
}
}
else
{
//Remove collection config member
var fcs1 = Configuration.CSVRecordFieldConfigurations.Where(o => o.FieldName == fqn).ToArray();
foreach (var fc in fcs1)
{
Configuration.CSVRecordFieldConfigurations.Remove(fc);
}
//Remove any unused config
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(itemType))
{
var fcs = Configuration.CSVRecordFieldConfigurations.Where(o => o.DeclaringMember == "{0}.{1}".FormatString(fullyQualifiedMemberName, pd.Name) &&
o.ArrayIndex != null && (o.ArrayIndex <minumum || o.ArrayIndex> maximum)).ToArray();
foreach (var fc in fcs)
{
Configuration.CSVRecordFieldConfigurations.Remove(fc);
}
}
for (int index = minumum; index <= maximum; index++)
{
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(itemType))
{
var fc = Configuration.CSVRecordFieldConfigurations.Where(o => o.DeclaringMember == "{0}.{1}".FormatString(fullyQualifiedMemberName, pd.Name) &&
o.ArrayIndex != null && o.ArrayIndex == index).FirstOrDefault();
if (fc != null)
{
continue;
}
Type pt = pd.PropertyType.GetUnderlyingType();
if (pt != typeof(object) && !pt.IsSimple())
{
}
else
{
int fieldPosition = 0;
fieldPosition = Configuration.CSVRecordFieldConfigurations.Count > 0 ? Configuration.CSVRecordFieldConfigurations.Max(f => f.FieldPosition) : 0;
fieldPosition++;
ChoCSVRecordFieldConfiguration obj = Configuration.NewFieldConfiguration(ref fieldPosition, fullyQualifiedMemberName, pd, index, displayName);
//if (!CSVRecordFieldConfigurations.Any(c => c.Name == (declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name))))
Configuration.CSVRecordFieldConfigurations.Add(obj);
}
}
}
}
}
return(this);
}
private bool FillRecord(object rec, Tuple <int, string> pair)
{
int lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
object fieldValue = null;
string[] fieldValues = (from x in line.Split(Configuration.Delimiter, Configuration.StringSplitOptions, Configuration.QuoteChar)
select x).ToArray();
if (Configuration.ColumnCountStrict)
{
if (fieldValues.Length != Configuration.RecordFieldConfigurations.Count)
{
throw new ChoParserException("Incorrect number of field values found at line [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.RecordFieldConfigurations.Count, fieldValues.Length, pair.Item1));
}
}
Dictionary <string, string> _fieldNameValues = ToFieldNameValues(fieldValues);
ValidateLine(pair.Item1, fieldValues);
ChoCSVRecordFieldConfiguration fieldConfig = null;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
if (Configuration.CSVFileHeaderConfiguration.HasHeaderRecord)
{
if (_fieldNameValues.ContainsKey(fieldConfig.FieldName))
{
fieldValue = _fieldNameValues[fieldConfig.FieldName];
}
else if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field header found.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (fieldConfig.FieldPosition - 1 < fieldValues.Length)
{
fieldValue = fieldValues[fieldConfig.FieldPosition - 1];
}
else if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing field value for {0} [Position: {1}] field.".FormatString(fieldConfig.FieldName, fieldConfig.FieldPosition));
}
}
fieldValue = CleanFieldValue(fieldConfig, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (!ignoreFieldValue)
{
if (rec is ExpandoObject)
{
if (fieldConfig.FieldType != typeof(string))
{
fieldValue = ChoConvert.ConvertTo(fieldValue, fieldConfig.FieldType, Configuration.Culture);
}
var x = rec as IDictionary <string, Object>;
x.Add(kvp.Key, fieldValue);
}
else
{
if (ChoType.HasProperty(rec.GetType(), kvp.Key))
{
ChoType.ConvertNSetMemberValue(rec, kvp.Key, fieldValue);
fieldValue = ChoType.GetMemberValue(rec, kvp.Key);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
ChoValidator.ValididateFor(rec, kvp.Key);
}
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
ChoFallbackValueAttribute fbAttr = ChoTypeDescriptor.GetPropetyAttribute <ChoFallbackValueAttribute>(rec.GetType(), kvp.Key);
if (fbAttr != null)
{
if (!fbAttr.Value.IsNullOrDbNull())
{
ChoType.ConvertNSetMemberValue(rec, kvp.Key, fbAttr.Value);
}
}
else
{
throw;
}
}
catch
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else if (fieldConfig.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
throw;
}
}
else
{
throw;
}
}
}
}
return(true);
}
private string ToText(object rec)
{
StringBuilder msg = new StringBuilder();
string fieldValue = null;
ChoCSVRecordFieldConfiguration fieldConfig = null;
//if (Configuration.ColumnCountStrict)
// CheckColumnsStrict(rec);
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
fieldValue = null;
if (Configuration.ThrowAndStopOnMissingField)
{
if (rec is ExpandoObject)
{
var x = rec as IDictionary <string, Object>;
if (!x.Keys.Contains(fieldConfig.FieldName, Configuration.CSVFileHeaderConfiguration.StringComparer))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (!ChoType.HasProperty(rec.GetType(), kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
}
//if (rec is ExpandoObject)
//{
// var x = rec as IDictionary<string, Object>;
// string key = x.Keys.Where(k => Configuration.CSVFileHeaderConfiguration.StringComparer.Compare(fieldConfig.FieldName, k) == 0).FirstOrDefault();
// if (!x.Keys.Select(.Contains(fieldConfig.FieldName, Configuration.CSVFileHeaderConfiguration.StringComparer))
// throw new ChoParserException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
//}
//else
//{
// if (!ChoType.HasProperty(rec.GetType(), kvp.Key))
// throw new ChoParserException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
//}
//fieldValue = CleanFieldValue(fieldConfig, fieldValue as string);
//if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
// continue;
//object value = ERPSType.GetMemberValue(RecordObject, member.Item1);
//if (value != null)
// value = ERPSConvert.ConvertTo(value, member.Item1, typeof(string), null);
//if (value == null)
// value = String.Empty;
//fieldValue = HasFieldsEnclosedInQuotes ? "\"{0}\"".FormatString(value.ToString()) : value.ToString();
//if (firstColumn)
//{
// msg.Append(NormalizeFieldValue(member.Item1.FullName(), member.Item2 as ERPSFileRecordFieldAttribute, fieldValue, Delimiter));
// firstColumn = false;
//}
//else
// msg.AppendFormat("{0}{1}", Delimiter, NormalizeFieldValue(member.Item1.FullName(), member.Item2 as ERPSFileRecordFieldAttribute, fieldValue, Delimiter));
}
return(msg.ToString());
}
