private string CleanFieldValue(ChoFixedLengthRecordFieldConfiguration config, string fieldValue) { if (fieldValue.IsNull()) { return(fieldValue); } if (fieldValue != null) { switch (config.FieldValueTrimOption) { case ChoFieldValueTrimOption.Trim: fieldValue = fieldValue.Trim(); break; case ChoFieldValueTrimOption.TrimStart: fieldValue = fieldValue.TrimStart(); break; case ChoFieldValueTrimOption.TrimEnd: fieldValue = fieldValue.TrimEnd(); break; } } if (config.Size != null) { if (fieldValue.Length > config.Size.Value) { if (!config.Truncate) { throw new ChoParserException("Incorrect field value length found for '{0}' member [Expected: {1}, Actual: {2}].".FormatString(config.FieldName, config.Size.Value, fieldValue.Length)); } else { fieldValue = fieldValue.Substring(0, config.Size.Value); } } } if (config.QuoteField != null && config.QuoteField.Value && fieldValue.StartsWith(@"""") && fieldValue.EndsWith(@"""")) { return(fieldValue.Substring(1, fieldValue.Length - 2)); } //else if ((fieldValue.Contains(Configuration.Delimiter) // || fieldValue.Contains(Configuration.EOLDelimiter)) && fieldValue.StartsWith(@"""") && fieldValue.EndsWith(@"""")) // return fieldValue.Substring(1, fieldValue.Length - 2); else { return(fieldValue); } }
private void DiscoverRecordFields(Type recordType) { if (!IsDynamicObject) { FixedLengthRecordFieldConfigurations.Clear(); foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType).Where(pd => pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().Any())) { //if (!pd.PropertyType.IsSimple()) // throw new ChoRecordConfigurationException("Property '{0}' is not a simple type.".FormatString(pd.Name)); var obj = new ChoFixedLengthRecordFieldConfiguration(pd.Name, pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().First()); obj.FieldType = pd.PropertyType; FixedLengthRecordFieldConfigurations.Add(obj); } } }
private string[] GetHeaders(string line) { List <string> headers = new List <string>(); if (Configuration.FileHeaderConfiguration.HasHeaderRecord) { //Fields are specified, load them if (Configuration.RecordFieldConfigurationsDict.Count > 0) { string fieldValue = null; ChoFixedLengthRecordFieldConfiguration fieldConfig = null; foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict) { fieldValue = null; fieldConfig = kvp.Value; if (fieldConfig.StartIndex + fieldConfig.Size > line.Length) { if (Configuration.ColumnCountStrict) { throw new ChoParserException("Missing '{0}' field.".FormatString(kvp.Key)); } } else { fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value); } fieldValue = CleanFieldValue(fieldConfig, typeof(object), fieldValue as string); headers.Add(fieldValue); } } else { if (line.Length != Configuration.RecordLength) { throw new ChoParserException("Incorrect header length [Length: {0}] found. Expected header length: {1}".FormatString(line.Length, Configuration.RecordLength)); } } } else { } return(headers.ToArray()); }
public static void Write(ChoFixedLengthWriter <dynamic> w, IDataReader dr) { ChoGuard.ArgumentNotNull(w, "Writer"); ChoGuard.ArgumentNotNull(dr, "DataReader"); DataTable schemaTable = dr.GetSchemaTable(); w.Configuration.UseNestedKeyFormat = false; w.Configuration.FixedLengthRecordFieldConfigurations.Clear(); //int ordinal = 0; if (w.Configuration.FixedLengthRecordFieldConfigurations.IsNullOrEmpty()) { string colName = null; Type colType = null; int startIndex = 0; int fieldLength = 0; foreach (DataRow row in schemaTable.Rows) { colName = row["ColumnName"].CastTo <string>(); colType = row["DataType"] as Type; //if (!colType.IsSimple()) continue; fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(colType); var obj = new ChoFixedLengthRecordFieldConfiguration(colName, startIndex, fieldLength); w.Configuration.FixedLengthRecordFieldConfigurations.Add(obj); startIndex += fieldLength; } } var ordinals = w.Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(c => c.Name, c => dr.HasColumn(c.Name) ? dr.GetOrdinal(c.Name) : -1); while (dr.Read()) { dynamic expando = new ExpandoObject(); var expandoDic = (IDictionary <string, object>)expando; foreach (var fc in ordinals) { expandoDic.Add(fc.Key, fc.Value == -1 ? null : dr[fc.Value]); } w.Write(expando); } }
public void Write(IDataReader dr) { ChoGuard.ArgumentNotNull(dr, "DataReader"); DataTable schemaTable = dr.GetSchemaTable(); dynamic expando = new ExpandoObject(); var expandoDic = (IDictionary <string, object>)expando; Configuration.UseNestedKeyFormat = false; //int ordinal = 0; if (Configuration.FixedLengthRecordFieldConfigurations.IsNullOrEmpty()) { string colName = null; Type colType = null; int startIndex = 0; int fieldLength = 0; foreach (DataRow row in schemaTable.Rows) { colName = row["ColumnName"].CastTo <string>(); colType = row["DataType"] as Type; //if (!colType.IsSimple()) continue; fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(colType); var obj = new ChoFixedLengthRecordFieldConfiguration(colName, startIndex, fieldLength); Configuration.FixedLengthRecordFieldConfigurations.Add(obj); startIndex += fieldLength; } } while (dr.Read()) { expandoDic.Clear(); foreach (var fc in Configuration.FixedLengthRecordFieldConfigurations) { expandoDic.Add(fc.Name, dr[fc.Name]); } Write(expando); } }
public static void Write(ChoFixedLengthWriter <dynamic> w, DataTable dt) { ChoGuard.ArgumentNotNull(w, "Writer"); ChoGuard.ArgumentNotNull(dt, "DataTable"); DataTable schemaTable = dt; w.Configuration.UseNestedKeyFormat = false; if (w.Configuration.FixedLengthRecordFieldConfigurations.IsNullOrEmpty()) { string colName = null; Type colType = null; int startIndex = 0; int fieldLength = 0; foreach (DataColumn col in schemaTable.Columns) { colName = col.ColumnName; colType = col.DataType; //if (!colType.IsSimple()) continue; fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(colType); var obj = new ChoFixedLengthRecordFieldConfiguration(colName, startIndex, fieldLength); w.Configuration.FixedLengthRecordFieldConfigurations.Add(obj); startIndex += fieldLength; } } foreach (DataRow row in dt.Rows) { dynamic expando = new ExpandoObject(); var expandoDic = (IDictionary <string, object>)expando; foreach (var fc in w.Configuration.FixedLengthRecordFieldConfigurations) { expandoDic.Add(fc.Name, row[fc.Name]); } w.Write(expando); } }
public override void Validate(object state) { base.Validate(state); string line = null; string[] fieldNames = null; if (state is Tuple <long, string> ) { line = ((Tuple <long, string>)state).Item2; } else { fieldNames = state as string[]; } if (RecordLength <= 0 && line != null) { RecordLength = line.Length; } //Validate Header if (FileHeaderConfiguration != null) { if (FileHeaderConfiguration.FillChar != null) { if (FileHeaderConfiguration.FillChar.Value == ChoCharEx.NUL) { throw new ChoRecordConfigurationException("Invalid '{0}' FillChar specified.".FormatString(FileHeaderConfiguration.FillChar)); } if (EOLDelimiter.Contains(FileHeaderConfiguration.FillChar.Value)) { throw new ChoRecordConfigurationException("FillChar [{0}] can't be one of EOLDelimiter characters [{1}]".FormatString(FileHeaderConfiguration.FillChar.Value, EOLDelimiter)); } if (Comments != null) { if ((from comm in Comments where comm.Contains(FileHeaderConfiguration.FillChar.Value.ToString()) select comm).Any()) { throw new ChoRecordConfigurationException("One of the Comments contains FillChar. Not allowed."); } } } } //string[] headers = state as string[]; if (AutoDiscoverColumns && FixedLengthRecordFieldConfigurations.Count == 0 /*&& headers != null*/) { if (RecordType != null && !IsDynamicObject && ChoTypeDescriptor.GetProperties(RecordType).Where(pd => pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().Any()).Any()) { int startIndex = 0; int size = 0; foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(RecordType).Where(pd => pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().Any())) { //if (!pd.PropertyType.IsSimple()) // throw new ChoRecordConfigurationException("Property '{0}' is not a simple type.".FormatString(pd.Name)); if (FixedLengthFieldDefaultSizeConfiguation == null) { size = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(pd.PropertyType); } else { size = FixedLengthFieldDefaultSizeConfiguation.GetSize(pd.PropertyType); } var obj = new ChoFixedLengthRecordFieldConfiguration(pd.Name, startIndex, size); obj.FieldType = pd.PropertyType; FixedLengthRecordFieldConfigurations.Add(obj); startIndex += size; } //RecordLength = startIndex; } else if (!line.IsNullOrEmpty()) { int index = 0; if (IsDynamicObject) { foreach (var item in DiscoverColumns(line)) { var obj = new ChoFixedLengthRecordFieldConfiguration(FileHeaderConfiguration.HasHeaderRecord ? item.Item1 : "Column{0}".FormatString(++index), item.Item2, item.Item3); FixedLengthRecordFieldConfigurations.Add(obj); } } else { Tuple <string, int, int>[] tuples = DiscoverColumns(line); foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(RecordType)) { if (index < tuples.Length) { var obj = new ChoFixedLengthRecordFieldConfiguration(FileHeaderConfiguration.HasHeaderRecord ? tuples[index].Item1 : pd.Name, tuples[index].Item2, tuples[index].Item3); FixedLengthRecordFieldConfigurations.Add(obj); index++; } else { break; } } } } else if (!fieldNames.IsNullOrEmpty()) { int startIndex = 0; int fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(typeof(string)); foreach (string fn in fieldNames) { var obj = new ChoFixedLengthRecordFieldConfiguration(fn, startIndex, fieldLength); FixedLengthRecordFieldConfigurations.Add(obj); startIndex += fieldLength; } } } if (FixedLengthRecordFieldConfigurations.Count == 0) { throw new ChoRecordConfigurationException("No record fields specified."); } //Derive record length from fields if (RecordLength <= 0) { int maxStartIndex = FixedLengthRecordFieldConfigurations.Max(f => f.StartIndex); int maxSize = FixedLengthRecordFieldConfigurations.Where(f => f.StartIndex == maxStartIndex).Max(f1 => f1.Size.Value); var fc = FixedLengthRecordFieldConfigurations.Where(f => f.StartIndex == maxStartIndex && f.Size.Value == maxSize).FirstOrDefault(); if (fc != null) { RecordLength = fc.StartIndex + fc.Size.Value; } } if (RecordLength <= 0) { throw new ChoRecordConfigurationException("RecordLength must be > 0"); } //Check if any field has empty names if (FixedLengthRecordFieldConfigurations.Where(i => i.FieldName.IsNullOrWhiteSpace()).Count() > 0) { throw new ChoRecordConfigurationException("Some fields has empty field name specified."); } //Check field names for duplicate string[] dupFields = FixedLengthRecordFieldConfigurations.GroupBy(i => i.FieldName, FileHeaderConfiguration.StringComparer) .Where(g => g.Count() > 1) .Select(g => g.Key).ToArray(); if (dupFields.Length > 0) { throw new ChoRecordConfigurationException("Duplicate field names [Name: {0}] specified to record fields.".FormatString(String.Join(",", dupFields))); } //Find duplicate fields with start index ChoFixedLengthRecordFieldConfiguration dupRecConfig = FixedLengthRecordFieldConfigurations.GroupBy(i => i.StartIndex).Where(g => g.Count() > 1).Select(g => g.FirstOrDefault()).FirstOrDefault(); if (dupRecConfig != null) { throw new ChoRecordConfigurationException("Found duplicate '{0}' record field with same start index.".FormatString(dupRecConfig.FieldName)); } //Check any overlapping fields specified foreach (var f in FixedLengthRecordFieldConfigurations) { if (f.StartIndex + f.Size.Value > RecordLength) { throw new ChoRecordConfigurationException("Found '{0}' record field out of bounds of record length.".FormatString(f.FieldName)); } } RecordFieldConfigurationsDict = FixedLengthRecordFieldConfigurations.OrderBy(i => i.StartIndex).Where(i => !i.Name.IsNullOrWhiteSpace()).ToDictionary(i => i.Name, FileHeaderConfiguration.StringComparer); RecordFieldConfigurationsDict2 = FixedLengthRecordFieldConfigurations.OrderBy(i => i.StartIndex).Where(i => !i.FieldName.IsNullOrWhiteSpace()).ToDictionary(i => i.FieldName, FileHeaderConfiguration.StringComparer); //Validate each record field foreach (var fieldConfig in FixedLengthRecordFieldConfigurations) { fieldConfig.Validate(this); } if (!FileHeaderConfiguration.HasHeaderRecord) { } else { } LoadNCacheMembers(FixedLengthRecordFieldConfigurations); }
private bool ToText(long index, object rec, out string recText) { if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType)) { rec = ChoActivator.CreateInstance(Configuration.RecordType, rec); } recText = null; StringBuilder msg = new StringBuilder(); object fieldValue = null; string fieldText = null; ChoFixedLengthRecordFieldConfiguration fieldConfig = null; if (Configuration.ColumnCountStrict) { CheckColumnsStrict(rec); } //bool firstColumn = true; PropertyInfo pi = null; object rootRec = rec; IDictionary <string, Object> dict = null; foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict) { fieldConfig = kvp.Value; fieldValue = null; fieldText = String.Empty; if (Configuration.PIDict != null) { Configuration.PIDict.TryGetValue(kvp.Key, out pi); } rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec); dict = rec.ToDynamicObject() as IDictionary <string, Object>; if (Configuration.IsDynamicObject) { dict = dict.Flatten().ToDictionary(); } if (Configuration.ThrowAndStopOnMissingField) { if (Configuration.IsDynamicObject) { if (!dict.ContainsKey(kvp.Key)) { throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName)); } } else { if (pi == null) { throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName)); } } } try { if (Configuration.IsDynamicObject) { fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture); if (kvp.Value.FieldType == null) { if (fieldValue == null) { kvp.Value.FieldType = typeof(string); } else { kvp.Value.FieldType = fieldValue.GetType(); } } } else { if (pi != null) { fieldValue = ChoType.GetPropertyValue(rec, pi); if (kvp.Value.FieldType == null) { kvp.Value.FieldType = pi.PropertyType; } } else { kvp.Value.FieldType = typeof(string); } } //Discover default value, use it if null if (fieldValue == null) { if (fieldConfig.IsDefaultValueSpecified) { fieldValue = fieldConfig.DefaultValue; } } if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue)) { return(false); } if (fieldConfig.ValueConverter != null) { fieldValue = fieldConfig.ValueConverter(fieldValue); } else { rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); } if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue)) { return(false); } } catch (ChoParserException) { throw; } catch (ChoMissingRecordFieldException) { if (Configuration.ThrowAndStopOnMissingField) { throw; } } catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop) { throw; } try { if (Configuration.IsDynamicObject) { if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); } else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); } else { var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex); fieldValue = null; throw ex1; } } else if (pi != null) { if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); } else { var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex); fieldValue = null; throw ex1; } } else { var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex); fieldValue = null; throw ex1; } } catch (Exception innerEx) { if (ex == innerEx.InnerException) { if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue) { continue; } else { if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex)) { throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex); } } } else { throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx); } } } if (fieldValue == null) { fieldText = String.Empty; } else { fieldText = fieldValue.ToString(); } msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType), GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false, kvp.Value.NullValue, kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType))); } recText = msg.ToString(); return(true); }
private ChoFixedLengthWriter <T> WithField(string name, int startIndex, int size, Type fieldType = null, bool?quoteField = null, char?fillChar = null, ChoFieldValueJustification?fieldValueJustification = null, bool truncate = true, string fieldName = null, Func <object, object> valueConverter = null, Func <dynamic, object> valueSelector = null, object defaultValue = null, object fallbackValue = null, string fullyQualifiedMemberName = null, string formatText = null, string nullValue = null) { if (!name.IsNullOrEmpty()) { if (!_clearFields) { ClearFields(); Configuration.MapRecordFields(Configuration.RecordType); } if (fieldName.IsNullOrWhiteSpace()) { fieldName = name; } string fnTrim = name.NTrim(); ChoFixedLengthRecordFieldConfiguration fc = null; PropertyDescriptor pd = null; if (Configuration.FixedLengthRecordFieldConfigurations.Any(o => o.Name == fnTrim)) { fc = Configuration.FixedLengthRecordFieldConfigurations.Where(o => o.Name == fnTrim).First(); Configuration.FixedLengthRecordFieldConfigurations.Remove(fc); } else { pd = ChoTypeDescriptor.GetNestedProperty(typeof(T), fullyQualifiedMemberName.IsNullOrWhiteSpace() ? name : fullyQualifiedMemberName); } var nfc = new ChoFixedLengthRecordFieldConfiguration(name.Trim(), startIndex, size) { FieldType = fieldType, QuoteField = quoteField, FillChar = fillChar, FieldValueJustification = fieldValueJustification, Truncate = truncate, FieldName = fieldName.IsNullOrWhiteSpace() ? name : fieldName, ValueConverter = valueConverter, ValueSelector = valueSelector, DefaultValue = defaultValue, FallbackValue = fallbackValue, FormatText = formatText, NullValue = nullValue }; if (fullyQualifiedMemberName.IsNullOrWhiteSpace()) { nfc.PropertyDescriptor = fc != null ? fc.PropertyDescriptor : pd; nfc.DeclaringMember = fc != null ? fc.DeclaringMember : fullyQualifiedMemberName; } else { pd = ChoTypeDescriptor.GetNestedProperty(typeof(T), fullyQualifiedMemberName); nfc.PropertyDescriptor = pd; nfc.DeclaringMember = fullyQualifiedMemberName; } if (pd != null) { if (nfc.FieldType == null) { nfc.FieldType = pd.PropertyType; } } Configuration.FixedLengthRecordFieldConfigurations.Add(nfc); } return(this); }
private string[] GetHeaders(string line) { string[] headers = null; if (Configuration.FileHeaderConfiguration.HasHeaderRecord && !Configuration.FileHeaderConfiguration.IgnoreHeader) { //Fields are specified, load them if (Configuration.RecordFieldConfigurationsDict.Count > 0) { List <string> headersList = new List <string>(); string fieldValue = null; ChoFixedLengthRecordFieldConfiguration fieldConfig = null; foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict) { fieldValue = null; fieldConfig = kvp.Value; if (fieldConfig.StartIndex + fieldConfig.Size > line.Length) { if (Configuration.ColumnCountStrict) { throw new ChoParserException("Missing '{0}' field.".FormatString(kvp.Key)); } } else { fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value); } fieldValue = CleanFieldValue(fieldConfig, typeof(object), fieldValue as string); headersList.Add(fieldValue); } headers = headersList.ToArray(); List <string> newHeaders = new List <string>(); int index = 1; string newColName = null; foreach (string header in headers) { if (RaiseMapColumn(this, index, header, out newColName)) { newHeaders.Add(newColName); } else { newHeaders.Add(header); } index++; } headers = newHeaders.ToArray(); //Check for any empty column headers if (headers.Where(h => h.IsNullOrEmpty()).Any()) { if (!Configuration.FileHeaderConfiguration.IgnoreColumnsWithEmptyHeader) { throw new ChoParserException("At least one of the field header is empty."); } else { index = 0; newHeaders = new List <string>(); foreach (string header in headers) { if (header.IsNullOrWhiteSpace()) { newHeaders.Add("_Column{0}".FormatString(++index)); } else { newHeaders.Add(header); } } headers = newHeaders.ToArray(); } } Configuration.Context.Headers = headers; } else { if (line.Length != Configuration.RecordLength) { throw new ChoParserException("Incorrect header length [Length: {0}] found. Expected header length: {1}".FormatString(line.Length, Configuration.RecordLength)); } } } else { } return(headers); }
private string CleanFieldValue(ChoFixedLengthRecordFieldConfiguration config, Type fieldType, string fieldValue) { if (fieldValue == null) { return(fieldValue); } ChoFieldValueTrimOption fieldValueTrimOption = config.GetFieldValueTrimOptionForRead(fieldType); switch (fieldValueTrimOption) { case ChoFieldValueTrimOption.Trim: fieldValue = fieldValue.Trim(); break; case ChoFieldValueTrimOption.TrimStart: fieldValue = fieldValue.TrimStart(); break; case ChoFieldValueTrimOption.TrimEnd: fieldValue = fieldValue.TrimEnd(); break; } char startChar; char endChar; char quoteChar = Configuration.QuoteChar == '\0' ? '"' : Configuration.QuoteChar; if (fieldValue.Length >= 2) { startChar = fieldValue[0]; endChar = fieldValue[fieldValue.Length - 1]; if (config.QuoteField != null && config.QuoteField.Value && startChar == quoteChar && endChar == quoteChar) { return(fieldValue.Substring(1, fieldValue.Length - 2)); } } if (config.Size != null) { if (fieldValue.Length > config.Size.Value) { if (!config.Truncate) { throw new ChoParserException("Incorrect field value length found for '{0}' member [Expected: {1}, Actual: {2}].".FormatString(config.FieldName, config.Size.Value, fieldValue.Length)); } else { if (fieldValueTrimOption == ChoFieldValueTrimOption.TrimStart) { fieldValue = fieldValue.Right(config.Size.Value); } else { fieldValue = fieldValue.Substring(0, config.Size.Value); } } } } if (config.NullValue != null) { if (String.Compare(config.NullValue, fieldValue, true) == 0) { fieldValue = null; } } return(fieldValue); }
private bool FillRecord(object rec, Tuple <long, string> pair) { long lineNo; string line; lineNo = pair.Item1; line = pair.Item2; if (line.Length != Configuration.RecordLength) { throw new ChoParserException("Incorrect record length [Length: {0}] found. Expected record length: {1}".FormatString(line.Length, Configuration.RecordLength)); } object fieldValue = null; ChoFixedLengthRecordFieldConfiguration fieldConfig = null; PropertyInfo pi = null; object rootRec = rec; foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict) { fieldValue = null; fieldConfig = kvp.Value; if (Configuration.PIDict != null) { Configuration.PIDict.TryGetValue(kvp.Key, out pi); } rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec); try { if (fieldConfig.StartIndex + fieldConfig.Size > line.Length) { if (Configuration.ColumnCountStrict) { throw new ChoParserException("Missing '{0}' field value.".FormatString(kvp.Key)); } } else { fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value); } if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { if (kvp.Value.FieldType == null) { kvp.Value.FieldType = Configuration.MaxScanRows == -1 ? DiscoverFieldType(fieldValue as string, Configuration) : typeof(string); } } else { if (pi != null) { kvp.Value.FieldType = pi.PropertyType; } else { kvp.Value.FieldType = typeof(string); } } fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string); if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue)) { continue; } bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue); if (ignoreFieldValue) { fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null; } if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { var dict = rec as IDictionary <string, Object>; dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture); if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } } else { if (Configuration.SupportsMultiRecordTypes) { ChoType.TryGetProperty(rec.GetType(), kvp.Key, out pi); fieldConfig.PI = pi; fieldConfig.PropConverters = ChoTypeDescriptor.GetTypeConverters(fieldConfig.PI); fieldConfig.PropConverterParams = ChoTypeDescriptor.GetTypeConverterParams(fieldConfig.PI); } if (pi != null) { rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture); } else if (!Configuration.SupportsMultiRecordTypes) { throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec))); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } } if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue)) { return(false); } } catch (ChoParserException) { Reader.IsValid = false; throw; } catch (ChoMissingRecordFieldException) { Reader.IsValid = false; if (Configuration.ThrowAndStopOnMissingField) { throw; } } catch (Exception ex) { Reader.IsValid = false; ChoETLFramework.HandleException(ref ex); if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop) { throw; } try { if (Configuration.IsDynamicObject) { var dict = rec as IDictionary <string, Object>; if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (ex is ValidationException) { throw; } else { throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } else if (pi != null) { if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (ex is ValidationException) { throw; } else { throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } else { throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } catch (Exception innerEx) { if (ex == innerEx.InnerException || ex is ValidationException) { if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue) { continue; } else { if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex)) { if (ex is ValidationException) { throw; } throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } } else { throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx); } } } } return(true); }
private void DiscoverRecordFields(Type recordType, string declaringMember, bool optIn = false) { if (!recordType.IsDynamicType()) { Type pt = null; int startIndex = 0; int size = 0; if (optIn) //ChoTypeDescriptor.GetProperties(recordType).Where(pd => pd.Attributes.OfType<ChoFixedLengthRecordFieldAttribute>().Any()).Any()) { foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType)) { pt = pd.PropertyType.GetUnderlyingType(); if (!pt.IsSimple() && !typeof(IEnumerable).IsAssignableFrom(pt)) { DiscoverRecordFields(pt, declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), optIn); } else if (pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().Any()) { var obj = new ChoFixedLengthRecordFieldConfiguration(pd.Name, pd.Attributes.OfType <ChoFixedLengthRecordFieldAttribute>().First(), pd.Attributes.OfType <Attribute>().ToArray()); obj.FieldType = pt; obj.PropertyDescriptor = pd; obj.DeclaringMember = declaringMember == null ? null : "{0}.{1}".FormatString(declaringMember, pd.Name); if (!FixedLengthRecordFieldConfigurations.Any(c => c.Name == pd.Name)) { FixedLengthRecordFieldConfigurations.Add(obj); } } } } else { foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(recordType)) { pt = pd.PropertyType.GetUnderlyingType(); if (pt != typeof(object) && !pt.IsSimple() && !typeof(IEnumerable).IsAssignableFrom(pt)) { DiscoverRecordFields(pt, declaringMember == null ? pd.Name : "{0}.{1}".FormatString(declaringMember, pd.Name), optIn); } else { if (FixedLengthFieldDefaultSizeConfiguation == null) { size = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(pd.PropertyType); } else { size = FixedLengthFieldDefaultSizeConfiguation.GetSize(pd.PropertyType); } var obj = new ChoFixedLengthRecordFieldConfiguration(pd.Name, startIndex, size); obj.FieldType = pt; obj.PropertyDescriptor = pd; obj.DeclaringMember = declaringMember == null ? null : "{0}.{1}".FormatString(declaringMember, pd.Name); StringLengthAttribute slAttr = pd.Attributes.OfType <StringLengthAttribute>().FirstOrDefault(); if (slAttr != null && slAttr.MaximumLength > 0) { obj.Size = slAttr.MaximumLength; } DisplayAttribute dpAttr = pd.Attributes.OfType <DisplayAttribute>().FirstOrDefault(); if (dpAttr != null) { if (!dpAttr.ShortName.IsNullOrWhiteSpace()) { obj.FieldName = dpAttr.ShortName; } else if (!dpAttr.Name.IsNullOrWhiteSpace()) { obj.FieldName = dpAttr.Name; } } DisplayFormatAttribute dfAttr = pd.Attributes.OfType <DisplayFormatAttribute>().FirstOrDefault(); if (dfAttr != null && !dfAttr.DataFormatString.IsNullOrWhiteSpace()) { obj.FormatText = dfAttr.DataFormatString; } if (dfAttr != null && !dfAttr.NullDisplayText.IsNullOrWhiteSpace()) { obj.NullValue = dfAttr.NullDisplayText; } if (!FixedLengthRecordFieldConfigurations.Any(c => c.Name == pd.Name)) { FixedLengthRecordFieldConfigurations.Add(obj); } startIndex += size; } } } } }
private bool ToText(int index, object rec, out string recText) { recText = null; StringBuilder msg = new StringBuilder(); object fieldValue = null; string fieldText = null; Type fieldType = null; ChoFixedLengthRecordFieldConfiguration fieldConfig = null; if (Configuration.ColumnCountStrict) CheckColumnsStrict(rec); //bool firstColumn = true; foreach (KeyValuePair<string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict) { fieldType = null; fieldConfig = kvp.Value; fieldValue = null; fieldText = String.Empty; if (Configuration.ThrowAndStopOnMissingField) { if (rec is ExpandoObject) { var dict = rec as IDictionary<string, Object>; if (!dict.Keys.Contains(kvp.Key, Configuration.FileHeaderConfiguration.StringComparer)) throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName)); } else { if (!ChoType.HasProperty(rec.GetType(), kvp.Key)) throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName)); } } try { if (rec is ExpandoObject) { IDictionary<string, Object> dict = rec as IDictionary<string, Object>; fieldValue = dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture); fieldType = kvp.Value.FieldType; } else { if (ChoType.HasProperty(rec.GetType(), kvp.Key)) { fieldValue = ChoType.GetPropertyValue(rec, kvp.Key); fieldType = ChoType.GetMemberType(rec.GetType(), kvp.Key); } } //Discover default value, use it if null if (fieldValue == null) fieldValue = rec.GetDefaultValue(kvp.Key, fieldConfig); if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue)) return false; fieldValue = rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, fieldValue); rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue)) return false; } catch (ChoParserException) { throw; } catch (ChoMissingRecordFieldException) { if (Configuration.ThrowAndStopOnMissingField) throw; } catch (Exception ex) { ChoETLFramework.HandleException(ex); if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop) throw; try { if (rec is ExpandoObject) { var dict = rec as IDictionary<string, Object>; if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); } else throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex); } else if (ChoType.HasProperty(rec.GetType(), kvp.Key) && rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); } else throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex); } catch (Exception innerEx) { if (ex == innerEx.InnerException) { if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue) { continue; } else { if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex)) throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex); } } else { throw new ChoParserException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx); } } } if (fieldValue == null) fieldText = String.Empty; else fieldText = fieldValue.ToString(); msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification, fieldType), GetFillChar(kvp.Value.FillChar, fieldType), false)); } recText = msg.ToString(); return true; }
internal ChoFixedLengthRecordFieldConfigurationMap(ChoFixedLengthRecordFieldConfiguration config) { _config = config; }
private bool FillRecord(object rec, Tuple <int, string> pair) { int lineNo; string line; lineNo = pair.Item1; line = pair.Item2; if (line.Length != Configuration.RecordLength) { throw new ChoParserException("Incorrect record length [Length: {0}] found. Expected record length: {1}".FormatString(line.Length, Configuration.RecordLength)); } object fieldValue = null; ChoFixedLengthRecordFieldConfiguration fieldConfig = null; foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict) { fieldValue = null; fieldConfig = kvp.Value; if (fieldConfig.StartIndex + fieldConfig.Size > line.Length) { if (Configuration.ColumnCountStrict) { throw new ChoParserException("Missing '{0}' field value.".FormatString(kvp.Key)); } } else { fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value); } fieldValue = CleanFieldValue(fieldConfig, fieldValue as string); if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue)) { return(false); } try { bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue); if (ignoreFieldValue) { fieldValue = null; } if (rec is ExpandoObject) { var dict = rec as IDictionary <string, Object>; dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture); if (ignoreFieldValue) { dict.AddOrUpdate(kvp.Key, fieldValue); } else { dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture); } dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else { if (ChoType.HasProperty(rec.GetType(), kvp.Key)) { rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture); if (!ignoreFieldValue) { rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture); } } else { throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec))); } rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue)) { return(false); } } catch (ChoParserException) { throw; } catch (ChoMissingRecordFieldException) { if (Configuration.ThrowAndStopOnMissingField) { throw; } } catch (Exception ex) { ChoETLFramework.HandleException(ex); if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop) { throw; } try { if (rec is ExpandoObject) { var dict = rec as IDictionary <string, Object>; if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else { throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } else if (ChoType.HasProperty(rec.GetType(), kvp.Key) && rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else { throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } catch (Exception innerEx) { if (ex == innerEx.InnerException) { if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue) { continue; } else { if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex)) { throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } } else { throw new ChoParserException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx); } } } } return(true); }