private string CleanFieldValue(ChoFixedLengthRecordFieldConfiguration config, string fieldValue)
{
if (fieldValue.IsNull())
{
return(fieldValue);
}
if (fieldValue != null)
{
switch (config.FieldValueTrimOption)
{
case ChoFieldValueTrimOption.Trim:
fieldValue = fieldValue.Trim();
break;
case ChoFieldValueTrimOption.TrimStart:
fieldValue = fieldValue.TrimStart();
break;
case ChoFieldValueTrimOption.TrimEnd:
fieldValue = fieldValue.TrimEnd();
break;
}
}
if (config.Size != null)
{
if (fieldValue.Length > config.Size.Value)
{
if (!config.Truncate)
{
throw new ChoParserException("Incorrect field value length found for '{0}' member [Expected: {1}, Actual: {2}].".FormatString(config.FieldName, config.Size.Value, fieldValue.Length));
}
else
{
fieldValue = fieldValue.Substring(0, config.Size.Value);
}
}
}
if (config.QuoteField != null && config.QuoteField.Value && fieldValue.StartsWith(@"""") && fieldValue.EndsWith(@""""))
{
return(fieldValue.Substring(1, fieldValue.Length - 2));
}
//else if ((fieldValue.Contains(Configuration.Delimiter)
// || fieldValue.Contains(Configuration.EOLDelimiter)) && fieldValue.StartsWith(@"""") && fieldValue.EndsWith(@""""))
// return fieldValue.Substring(1, fieldValue.Length - 2);
else
{
return(fieldValue);
}
}
private void DiscoverRecordFields(Type recordType)
{
if (!IsDynamicObject)
{
FixedLengthRecordFieldConfigurations.Clear();
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType).Where(pd => pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().Any()))
{
//if (!pd.PropertyType.IsSimple())
// throw new ChoRecordConfigurationException("Property '{0}' is not a simple type.".FormatString(pd.Name));
var obj = new ChoFixedLengthRecordFieldConfiguration(pd.Name, pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().First());
obj.FieldType = pd.PropertyType;
FixedLengthRecordFieldConfigurations.Add(obj);
}
}
}
private string[] GetHeaders(string line)
{
List <string> headers = new List <string>();
if (Configuration.FileHeaderConfiguration.HasHeaderRecord)
{
//Fields are specified, load them
if (Configuration.RecordFieldConfigurationsDict.Count > 0)
{
string fieldValue = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (fieldConfig.StartIndex + fieldConfig.Size > line.Length)
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing '{0}' field.".FormatString(kvp.Key));
}
}
else
{
fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value);
}
fieldValue = CleanFieldValue(fieldConfig, typeof(object), fieldValue as string);
headers.Add(fieldValue);
}
}
else
{
if (line.Length != Configuration.RecordLength)
{
throw new ChoParserException("Incorrect header length [Length: {0}] found. Expected header length: {1}".FormatString(line.Length, Configuration.RecordLength));
}
}
}
else
{
}
return(headers.ToArray());
}
public static void Write(ChoFixedLengthWriter <dynamic> w, IDataReader dr)
{
ChoGuard.ArgumentNotNull(w, "Writer");
ChoGuard.ArgumentNotNull(dr, "DataReader");
DataTable schemaTable = dr.GetSchemaTable();
w.Configuration.UseNestedKeyFormat = false;
w.Configuration.FixedLengthRecordFieldConfigurations.Clear();
//int ordinal = 0;
if (w.Configuration.FixedLengthRecordFieldConfigurations.IsNullOrEmpty())
{
string colName = null;
Type colType = null;
int startIndex = 0;
int fieldLength = 0;
foreach (DataRow row in schemaTable.Rows)
{
colName = row["ColumnName"].CastTo <string>();
colType = row["DataType"] as Type;
//if (!colType.IsSimple()) continue;
fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(colType);
var obj = new ChoFixedLengthRecordFieldConfiguration(colName, startIndex, fieldLength);
w.Configuration.FixedLengthRecordFieldConfigurations.Add(obj);
startIndex += fieldLength;
}
}
var ordinals = w.Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(c => c.Name, c => dr.HasColumn(c.Name) ? dr.GetOrdinal(c.Name) : -1);
while (dr.Read())
{
dynamic expando = new ExpandoObject();
var expandoDic = (IDictionary <string, object>)expando;
foreach (var fc in ordinals)
{
expandoDic.Add(fc.Key, fc.Value == -1 ? null : dr[fc.Value]);
}
w.Write(expando);
}
}
public void Write(IDataReader dr)
{
ChoGuard.ArgumentNotNull(dr, "DataReader");
DataTable schemaTable = dr.GetSchemaTable();
dynamic expando = new ExpandoObject();
var expandoDic = (IDictionary <string, object>)expando;
Configuration.UseNestedKeyFormat = false;
//int ordinal = 0;
if (Configuration.FixedLengthRecordFieldConfigurations.IsNullOrEmpty())
{
string colName = null;
Type colType = null;
int startIndex = 0;
int fieldLength = 0;
foreach (DataRow row in schemaTable.Rows)
{
colName = row["ColumnName"].CastTo <string>();
colType = row["DataType"] as Type;
//if (!colType.IsSimple()) continue;
fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(colType);
var obj = new ChoFixedLengthRecordFieldConfiguration(colName, startIndex, fieldLength);
Configuration.FixedLengthRecordFieldConfigurations.Add(obj);
startIndex += fieldLength;
}
}
while (dr.Read())
{
expandoDic.Clear();
foreach (var fc in Configuration.FixedLengthRecordFieldConfigurations)
{
expandoDic.Add(fc.Name, dr[fc.Name]);
}
Write(expando);
}
}
public static void Write(ChoFixedLengthWriter <dynamic> w, DataTable dt)
{
ChoGuard.ArgumentNotNull(w, "Writer");
ChoGuard.ArgumentNotNull(dt, "DataTable");
DataTable schemaTable = dt;
w.Configuration.UseNestedKeyFormat = false;
if (w.Configuration.FixedLengthRecordFieldConfigurations.IsNullOrEmpty())
{
string colName = null;
Type colType = null;
int startIndex = 0;
int fieldLength = 0;
foreach (DataColumn col in schemaTable.Columns)
{
colName = col.ColumnName;
colType = col.DataType;
//if (!colType.IsSimple()) continue;
fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(colType);
var obj = new ChoFixedLengthRecordFieldConfiguration(colName, startIndex, fieldLength);
w.Configuration.FixedLengthRecordFieldConfigurations.Add(obj);
startIndex += fieldLength;
}
}
foreach (DataRow row in dt.Rows)
{
dynamic expando = new ExpandoObject();
var expandoDic = (IDictionary <string, object>)expando;
foreach (var fc in w.Configuration.FixedLengthRecordFieldConfigurations)
{
expandoDic.Add(fc.Name, row[fc.Name]);
}
w.Write(expando);
}
}
public override void Validate(object state)
{
base.Validate(state);
string line = null;
string[] fieldNames = null;
if (state is Tuple <long, string> )
{
line = ((Tuple <long, string>)state).Item2;
}
else
{
fieldNames = state as string[];
}
if (RecordLength <= 0 && line != null)
{
RecordLength = line.Length;
}
//Validate Header
if (FileHeaderConfiguration != null)
{
if (FileHeaderConfiguration.FillChar != null)
{
if (FileHeaderConfiguration.FillChar.Value == ChoCharEx.NUL)
{
throw new ChoRecordConfigurationException("Invalid '{0}' FillChar specified.".FormatString(FileHeaderConfiguration.FillChar));
}
if (EOLDelimiter.Contains(FileHeaderConfiguration.FillChar.Value))
{
throw new ChoRecordConfigurationException("FillChar [{0}] can't be one of EOLDelimiter characters [{1}]".FormatString(FileHeaderConfiguration.FillChar.Value, EOLDelimiter));
}
if (Comments != null)
{
if ((from comm in Comments
where comm.Contains(FileHeaderConfiguration.FillChar.Value.ToString())
select comm).Any())
{
throw new ChoRecordConfigurationException("One of the Comments contains FillChar. Not allowed.");
}
}
}
}
//string[] headers = state as string[];
if (AutoDiscoverColumns &&
FixedLengthRecordFieldConfigurations.Count == 0 /*&& headers != null*/)
{
if (RecordType != null && !IsDynamicObject &&
ChoTypeDescriptor.GetProperties(RecordType).Where(pd => pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().Any()).Any())
{
int startIndex = 0;
int size = 0;
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(RecordType).Where(pd => pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().Any()))
{
//if (!pd.PropertyType.IsSimple())
// throw new ChoRecordConfigurationException("Property '{0}' is not a simple type.".FormatString(pd.Name));
if (FixedLengthFieldDefaultSizeConfiguation == null)
{
size = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(pd.PropertyType);
}
else
{
size = FixedLengthFieldDefaultSizeConfiguation.GetSize(pd.PropertyType);
}
var obj = new ChoFixedLengthRecordFieldConfiguration(pd.Name, startIndex, size);
obj.FieldType = pd.PropertyType;
FixedLengthRecordFieldConfigurations.Add(obj);
startIndex += size;
}
//RecordLength = startIndex;
}
else if (!line.IsNullOrEmpty())
{
int index = 0;
if (IsDynamicObject)
{
foreach (var item in DiscoverColumns(line))
{
var obj = new ChoFixedLengthRecordFieldConfiguration(FileHeaderConfiguration.HasHeaderRecord ? item.Item1 : "Column{0}".FormatString(++index), item.Item2, item.Item3);
FixedLengthRecordFieldConfigurations.Add(obj);
}
}
else
{
Tuple <string, int, int>[] tuples = DiscoverColumns(line);
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(RecordType))
{
if (index < tuples.Length)
{
var obj = new ChoFixedLengthRecordFieldConfiguration(FileHeaderConfiguration.HasHeaderRecord ? tuples[index].Item1 : pd.Name, tuples[index].Item2, tuples[index].Item3);
FixedLengthRecordFieldConfigurations.Add(obj);
index++;
}
else
{
break;
}
}
}
}
else if (!fieldNames.IsNullOrEmpty())
{
int startIndex = 0;
int fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(typeof(string));
foreach (string fn in fieldNames)
{
var obj = new ChoFixedLengthRecordFieldConfiguration(fn, startIndex, fieldLength);
FixedLengthRecordFieldConfigurations.Add(obj);
startIndex += fieldLength;
}
}
}
if (FixedLengthRecordFieldConfigurations.Count == 0)
{
throw new ChoRecordConfigurationException("No record fields specified.");
}
//Derive record length from fields
if (RecordLength <= 0)
{
int maxStartIndex = FixedLengthRecordFieldConfigurations.Max(f => f.StartIndex);
int maxSize = FixedLengthRecordFieldConfigurations.Where(f => f.StartIndex == maxStartIndex).Max(f1 => f1.Size.Value);
var fc = FixedLengthRecordFieldConfigurations.Where(f => f.StartIndex == maxStartIndex && f.Size.Value == maxSize).FirstOrDefault();
if (fc != null)
{
RecordLength = fc.StartIndex + fc.Size.Value;
}
}
if (RecordLength <= 0)
{
throw new ChoRecordConfigurationException("RecordLength must be > 0");
}
//Check if any field has empty names
if (FixedLengthRecordFieldConfigurations.Where(i => i.FieldName.IsNullOrWhiteSpace()).Count() > 0)
{
throw new ChoRecordConfigurationException("Some fields has empty field name specified.");
}
//Check field names for duplicate
string[] dupFields = FixedLengthRecordFieldConfigurations.GroupBy(i => i.FieldName, FileHeaderConfiguration.StringComparer)
.Where(g => g.Count() > 1)
.Select(g => g.Key).ToArray();
if (dupFields.Length > 0)
{
throw new ChoRecordConfigurationException("Duplicate field names [Name: {0}] specified to record fields.".FormatString(String.Join(",", dupFields)));
}
//Find duplicate fields with start index
ChoFixedLengthRecordFieldConfiguration dupRecConfig = FixedLengthRecordFieldConfigurations.GroupBy(i => i.StartIndex).Where(g => g.Count() > 1).Select(g => g.FirstOrDefault()).FirstOrDefault();
if (dupRecConfig != null)
{
throw new ChoRecordConfigurationException("Found duplicate '{0}' record field with same start index.".FormatString(dupRecConfig.FieldName));
}
//Check any overlapping fields specified
foreach (var f in FixedLengthRecordFieldConfigurations)
{
if (f.StartIndex + f.Size.Value > RecordLength)
{
throw new ChoRecordConfigurationException("Found '{0}' record field out of bounds of record length.".FormatString(f.FieldName));
}
}
RecordFieldConfigurationsDict = FixedLengthRecordFieldConfigurations.OrderBy(i => i.StartIndex).Where(i => !i.Name.IsNullOrWhiteSpace()).ToDictionary(i => i.Name, FileHeaderConfiguration.StringComparer);
RecordFieldConfigurationsDict2 = FixedLengthRecordFieldConfigurations.OrderBy(i => i.StartIndex).Where(i => !i.FieldName.IsNullOrWhiteSpace()).ToDictionary(i => i.FieldName, FileHeaderConfiguration.StringComparer);
//Validate each record field
foreach (var fieldConfig in FixedLengthRecordFieldConfigurations)
{
fieldConfig.Validate(this);
}
if (!FileHeaderConfiguration.HasHeaderRecord)
{
}
else
{
}
LoadNCacheMembers(FixedLengthRecordFieldConfigurations);
}
private bool ToText(long index, object rec, out string recText)
{
if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType))
{
rec = ChoActivator.CreateInstance(Configuration.RecordType, rec);
}
recText = null;
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
//bool firstColumn = true;
PropertyInfo pi = null;
object rootRec = rec;
IDictionary <string, Object> dict = null;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (Configuration.IsDynamicObject)
{
dict = dict.Flatten().ToDictionary();
}
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
if (!dict.ContainsKey(kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (pi == null)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (kvp.Value.FieldType == null)
{
if (fieldValue == null)
{
kvp.Value.FieldType = typeof(string);
}
else
{
kvp.Value.FieldType = fieldValue.GetType();
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
if (fieldConfig.IsDefaultValueSpecified)
{
fieldValue = fieldConfig.DefaultValue;
}
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else
{
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
{
fieldText = String.Empty;
}
else
{
fieldText = fieldValue.ToString();
}
msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField,
GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType),
GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false, kvp.Value.NullValue,
kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType)));
}
recText = msg.ToString();
return(true);
}
private ChoFixedLengthWriter <T> WithField(string name, int startIndex, int size, Type fieldType = null, bool?quoteField = null, char?fillChar = null, ChoFieldValueJustification?fieldValueJustification = null,
bool truncate = true, string fieldName = null, Func <object, object> valueConverter = null,
Func <dynamic, object> valueSelector = null,
object defaultValue = null, object fallbackValue = null,
string fullyQualifiedMemberName = null, string formatText = null, string nullValue = null)
{
if (!name.IsNullOrEmpty())
{
if (!_clearFields)
{
ClearFields();
Configuration.MapRecordFields(Configuration.RecordType);
}
if (fieldName.IsNullOrWhiteSpace())
{
fieldName = name;
}
string fnTrim = name.NTrim();
ChoFixedLengthRecordFieldConfiguration fc = null;
PropertyDescriptor pd = null;
if (Configuration.FixedLengthRecordFieldConfigurations.Any(o => o.Name == fnTrim))
{
fc = Configuration.FixedLengthRecordFieldConfigurations.Where(o => o.Name == fnTrim).First();
Configuration.FixedLengthRecordFieldConfigurations.Remove(fc);
}
else
{
pd = ChoTypeDescriptor.GetNestedProperty(typeof(T), fullyQualifiedMemberName.IsNullOrWhiteSpace() ? name : fullyQualifiedMemberName);
}
var nfc = new ChoFixedLengthRecordFieldConfiguration(name.Trim(), startIndex, size)
{
FieldType = fieldType,
QuoteField = quoteField,
FillChar = fillChar,
FieldValueJustification = fieldValueJustification,
Truncate = truncate,
FieldName = fieldName.IsNullOrWhiteSpace() ? name : fieldName,
ValueConverter = valueConverter,
ValueSelector = valueSelector,
DefaultValue = defaultValue,
FallbackValue = fallbackValue,
FormatText = formatText,
NullValue = nullValue
};
if (fullyQualifiedMemberName.IsNullOrWhiteSpace())
{
nfc.PropertyDescriptor = fc != null ? fc.PropertyDescriptor : pd;
nfc.DeclaringMember = fc != null ? fc.DeclaringMember : fullyQualifiedMemberName;
}
else
{
pd = ChoTypeDescriptor.GetNestedProperty(typeof(T), fullyQualifiedMemberName);
nfc.PropertyDescriptor = pd;
nfc.DeclaringMember = fullyQualifiedMemberName;
}
if (pd != null)
{
if (nfc.FieldType == null)
{
nfc.FieldType = pd.PropertyType;
}
}
Configuration.FixedLengthRecordFieldConfigurations.Add(nfc);
}
return(this);
}
private string[] GetHeaders(string line)
{
string[] headers = null;
if (Configuration.FileHeaderConfiguration.HasHeaderRecord && !Configuration.FileHeaderConfiguration.IgnoreHeader)
{
//Fields are specified, load them
if (Configuration.RecordFieldConfigurationsDict.Count > 0)
{
List <string> headersList = new List <string>();
string fieldValue = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (fieldConfig.StartIndex + fieldConfig.Size > line.Length)
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing '{0}' field.".FormatString(kvp.Key));
}
}
else
{
fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value);
}
fieldValue = CleanFieldValue(fieldConfig, typeof(object), fieldValue as string);
headersList.Add(fieldValue);
}
headers = headersList.ToArray();
List <string> newHeaders = new List <string>();
int index = 1;
string newColName = null;
foreach (string header in headers)
{
if (RaiseMapColumn(this, index, header, out newColName))
{
newHeaders.Add(newColName);
}
else
{
newHeaders.Add(header);
}
index++;
}
headers = newHeaders.ToArray();
//Check for any empty column headers
if (headers.Where(h => h.IsNullOrEmpty()).Any())
{
if (!Configuration.FileHeaderConfiguration.IgnoreColumnsWithEmptyHeader)
{
throw new ChoParserException("At least one of the field header is empty.");
}
else
{
index = 0;
newHeaders = new List <string>();
foreach (string header in headers)
{
if (header.IsNullOrWhiteSpace())
{
newHeaders.Add("_Column{0}".FormatString(++index));
}
else
{
newHeaders.Add(header);
}
}
headers = newHeaders.ToArray();
}
}
Configuration.Context.Headers = headers;
}
else
{
if (line.Length != Configuration.RecordLength)
{
throw new ChoParserException("Incorrect header length [Length: {0}] found. Expected header length: {1}".FormatString(line.Length, Configuration.RecordLength));
}
}
}
else
{
}
return(headers);
}
private string CleanFieldValue(ChoFixedLengthRecordFieldConfiguration config, Type fieldType, string fieldValue)
{
if (fieldValue == null)
{
return(fieldValue);
}
ChoFieldValueTrimOption fieldValueTrimOption = config.GetFieldValueTrimOptionForRead(fieldType);
switch (fieldValueTrimOption)
{
case ChoFieldValueTrimOption.Trim:
fieldValue = fieldValue.Trim();
break;
case ChoFieldValueTrimOption.TrimStart:
fieldValue = fieldValue.TrimStart();
break;
case ChoFieldValueTrimOption.TrimEnd:
fieldValue = fieldValue.TrimEnd();
break;
}
char startChar;
char endChar;
char quoteChar = Configuration.QuoteChar == '\0' ? '"' : Configuration.QuoteChar;
if (fieldValue.Length >= 2)
{
startChar = fieldValue[0];
endChar = fieldValue[fieldValue.Length - 1];
if (config.QuoteField != null && config.QuoteField.Value && startChar == quoteChar && endChar == quoteChar)
{
return(fieldValue.Substring(1, fieldValue.Length - 2));
}
}
if (config.Size != null)
{
if (fieldValue.Length > config.Size.Value)
{
if (!config.Truncate)
{
throw new ChoParserException("Incorrect field value length found for '{0}' member [Expected: {1}, Actual: {2}].".FormatString(config.FieldName, config.Size.Value, fieldValue.Length));
}
else
{
if (fieldValueTrimOption == ChoFieldValueTrimOption.TrimStart)
{
fieldValue = fieldValue.Right(config.Size.Value);
}
else
{
fieldValue = fieldValue.Substring(0, config.Size.Value);
}
}
}
}
if (config.NullValue != null)
{
if (String.Compare(config.NullValue, fieldValue, true) == 0)
{
fieldValue = null;
}
}
return(fieldValue);
}
private bool FillRecord(object rec, Tuple <long, string> pair)
{
long lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
if (line.Length != Configuration.RecordLength)
{
throw new ChoParserException("Incorrect record length [Length: {0}] found. Expected record length: {1}".FormatString(line.Length, Configuration.RecordLength));
}
object fieldValue = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
PropertyInfo pi = null;
object rootRec = rec;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
try
{
if (fieldConfig.StartIndex + fieldConfig.Size > line.Length)
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing '{0}' field value.".FormatString(kvp.Key));
}
}
else
{
fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value);
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = Configuration.MaxScanRows == -1 ? DiscoverFieldType(fieldValue as string, Configuration) : typeof(string);
}
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (Configuration.SupportsMultiRecordTypes)
{
ChoType.TryGetProperty(rec.GetType(), kvp.Key, out pi);
fieldConfig.PI = pi;
fieldConfig.PropConverters = ChoTypeDescriptor.GetTypeConverters(fieldConfig.PI);
fieldConfig.PropConverterParams = ChoTypeDescriptor.GetTypeConverterParams(fieldConfig.PI);
}
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else if (!Configuration.SupportsMultiRecordTypes)
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
Reader.IsValid = false;
throw;
}
catch (ChoMissingRecordFieldException)
{
Reader.IsValid = false;
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException || ex is ValidationException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
if (ex is ValidationException)
{
throw;
}
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
private void DiscoverRecordFields(Type recordType, string declaringMember, bool optIn = false)
{
if (!recordType.IsDynamicType())
{
Type pt = null;
int startIndex = 0;
int size = 0;
if (optIn) //ChoTypeDescriptor.GetProperties(recordType).Where(pd => pd.Attributes.OfType<ChoFixedLengthRecordFieldAttribute>().Any()).Any())
{
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType))
{
pt = pd.PropertyType.GetUnderlyingType();
if (!pt.IsSimple() && !typeof(IEnumerable).IsAssignableFrom(pt))
{
DiscoverRecordFields(pt, declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), optIn);
}
else if (pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().Any())
{
var obj = new ChoFixedLengthRecordFieldConfiguration(pd.Name, pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().First(), pd.Attributes.OfType <Attribute>().ToArray());
obj.FieldType = pt;
obj.PropertyDescriptor = pd;
obj.DeclaringMember = declaringMember == null ? null : "{0}.{1}".FormatString(declaringMember, pd.Name);
if (!FixedLengthRecordFieldConfigurations.Any(c => c.Name == pd.Name))
{
FixedLengthRecordFieldConfigurations.Add(obj);
}
}
}
}
else
{
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType))
{
pt = pd.PropertyType.GetUnderlyingType();
if (pt != typeof(object) && !pt.IsSimple() && !typeof(IEnumerable).IsAssignableFrom(pt))
{
DiscoverRecordFields(pt, declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), optIn);
}
else
{
if (FixedLengthFieldDefaultSizeConfiguation == null)
{
size = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(pd.PropertyType);
}
else
{
size = FixedLengthFieldDefaultSizeConfiguation.GetSize(pd.PropertyType);
}
var obj = new ChoFixedLengthRecordFieldConfiguration(pd.Name, startIndex, size);
obj.FieldType = pt;
obj.PropertyDescriptor = pd;
obj.DeclaringMember = declaringMember == null ? null : "{0}.{1}".FormatString(declaringMember, pd.Name);
StringLengthAttribute slAttr = pd.Attributes.OfType <StringLengthAttribute>().FirstOrDefault();
if (slAttr != null && slAttr.MaximumLength > 0)
{
obj.Size = slAttr.MaximumLength;
}
DisplayAttribute dpAttr = pd.Attributes.OfType <DisplayAttribute>().FirstOrDefault();
if (dpAttr != null)
{
if (!dpAttr.ShortName.IsNullOrWhiteSpace())
{
obj.FieldName = dpAttr.ShortName;
}
else if (!dpAttr.Name.IsNullOrWhiteSpace())
{
obj.FieldName = dpAttr.Name;
}
}
DisplayFormatAttribute dfAttr = pd.Attributes.OfType <DisplayFormatAttribute>().FirstOrDefault();
if (dfAttr != null && !dfAttr.DataFormatString.IsNullOrWhiteSpace())
{
obj.FormatText = dfAttr.DataFormatString;
}
if (dfAttr != null && !dfAttr.NullDisplayText.IsNullOrWhiteSpace())
{
obj.NullValue = dfAttr.NullDisplayText;
}
if (!FixedLengthRecordFieldConfigurations.Any(c => c.Name == pd.Name))
{
FixedLengthRecordFieldConfigurations.Add(obj);
}
startIndex += size;
}
}
}
}
}
private bool ToText(int index, object rec, out string recText)
{
recText = null;
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
Type fieldType = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
if (Configuration.ColumnCountStrict)
CheckColumnsStrict(rec);
//bool firstColumn = true;
foreach (KeyValuePair<string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldType = null;
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.ThrowAndStopOnMissingField)
{
if (rec is ExpandoObject)
{
var dict = rec as IDictionary<string, Object>;
if (!dict.Keys.Contains(kvp.Key, Configuration.FileHeaderConfiguration.StringComparer))
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
}
else
{
if (!ChoType.HasProperty(rec.GetType(), kvp.Key))
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
}
}
try
{
if (rec is ExpandoObject)
{
IDictionary<string, Object> dict = rec as IDictionary<string, Object>;
fieldValue = dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
fieldType = kvp.Value.FieldType;
}
else
{
if (ChoType.HasProperty(rec.GetType(), kvp.Key))
{
fieldValue = ChoType.GetPropertyValue(rec, kvp.Key);
fieldType = ChoType.GetMemberType(rec.GetType(), kvp.Key);
}
}
//Discover default value, use it if null
if (fieldValue == null)
fieldValue = rec.GetDefaultValue(kvp.Key, fieldConfig);
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
return false;
fieldValue = rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, fieldValue);
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
return false;
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
throw;
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
throw;
try
{
if (rec is ExpandoObject)
{
var dict = rec as IDictionary<string, Object>;
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
else if (ChoType.HasProperty(rec.GetType(), kvp.Key) && rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
else
{
throw new ChoParserException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
fieldText = String.Empty;
else
fieldText = fieldValue.ToString();
msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification, fieldType), GetFillChar(kvp.Value.FillChar, fieldType), false));
}
recText = msg.ToString();
return true;
}
internal ChoFixedLengthRecordFieldConfigurationMap(ChoFixedLengthRecordFieldConfiguration config)
{
_config = config;
}
private bool FillRecord(object rec, Tuple <int, string> pair)
{
int lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
if (line.Length != Configuration.RecordLength)
{
throw new ChoParserException("Incorrect record length [Length: {0}] found. Expected record length: {1}".FormatString(line.Length, Configuration.RecordLength));
}
object fieldValue = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (fieldConfig.StartIndex + fieldConfig.Size > line.Length)
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing '{0}' field value.".FormatString(kvp.Key));
}
}
else
{
fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value);
}
fieldValue = CleanFieldValue(fieldConfig, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
return(false);
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = null;
}
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture);
if (ignoreFieldValue)
{
dict.AddOrUpdate(kvp.Key, fieldValue);
}
else
{
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
if (ChoType.HasProperty(rec.GetType(), kvp.Key))
{
rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture);
if (!ignoreFieldValue)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (ChoType.HasProperty(rec.GetType(), kvp.Key) && rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoParserException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
