private bool FillRecord(ref object rec, Tuple <long, IDictionary <string, object> > pair) { long lineNo; lineNo = pair.Item1; var node = pair.Item2; if (Configuration.LiteParsing && Configuration.IsDynamicObject && rec is ChoDynamicObject) { ((ChoDynamicObject)rec).SetDictionary(node as IDictionary <string, object>); return(true); } fieldValue = null; fieldConfig = null; pi = null; object rootRec = rec; foreach (KeyValuePair <string, ChoParquetRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict) { if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { if (Configuration.IgnoredFields.Contains(kvp.Key)) { continue; } } fieldValue = null; fieldConfig = kvp.Value; if (Configuration.PIDict != null) { // if FieldName is set if (!string.IsNullOrEmpty(fieldConfig.FieldName)) { // match using FieldName Configuration.PIDict.TryGetValue(fieldConfig.FieldName, out pi); } if (pi == null) { // otherwise match usign the property name Configuration.PIDict.TryGetValue(kvp.Key, out pi); } } rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec); if (!node.ContainsKey(kvp.Value.FieldName)) { if (Configuration.ColumnCountStrict) { throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName)); } } else { fieldValue = node[kvp.Value.FieldName]; } if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue)) { continue; } try { if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { } else { if (pi != null) { kvp.Value.FieldType = pi.PropertyType; } else { kvp.Value.FieldType = typeof(string); } } object v1 = node; if (fieldConfig.CustomSerializer != null) { fieldValue = fieldConfig.CustomSerializer(v1); } else if (RaiseRecordFieldDeserialize(rec, pair.Item1, kvp.Key, ref v1)) { fieldValue = v1; } else if (fieldConfig.PropCustomSerializer != null) { fieldValue = ChoCustomSerializer.Deserialize(v1, fieldConfig.FieldType, fieldConfig.PropCustomSerializer, fieldConfig.PropCustomSerializerParams, Configuration.Culture, fieldConfig.Name); } else { if (!node.ContainsKey(kvp.Value.FieldName)) { if (Configuration.ColumnCountStrict) { throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName)); } } else { fieldValue = node[kvp.Value.FieldName]; } } if (fieldConfig.IgnoreFieldValueMode == null) { if (fieldValue.IsObjectNullOrEmpty() && fieldConfig.IsDefaultValueSpecified) { fieldValue = fieldConfig.DefaultValue; } } else { bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode); if (ignoreFieldValue && fieldConfig.IsDefaultValueSpecified) { fieldValue = fieldConfig.DefaultValue; } ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode); if (ignoreFieldValue) { continue; } } if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { var dict = rec as IDictionary <string, Object>; dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture); if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } } else { if (Configuration.SupportsMultiRecordTypes) { ChoType.TryGetProperty(rec.GetType(), kvp.Key, out pi); //*** TODO //fieldConfig.PI = pi; //fieldConfig.PropConverters = ChoTypeDescriptor.GetTypeConverters(fieldConfig.PI); //fieldConfig.PropConverterParams = ChoTypeDescriptor.GetTypeConverterParams(fieldConfig.PI); } if (pi != null) { if (Configuration.LiteParsing) { ChoType.SetPropertyValue(rec, fieldConfig.PI, fieldConfig.FieldType == null || fieldConfig.FieldType == typeof(string) ? fieldValue : Convert.ChangeType(fieldValue, fieldConfig.FieldType, Configuration.Culture)); } else { rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture); } } else if (RecordType.IsSimple()) { rec = ChoConvert.ConvertTo(fieldValue, RecordType, Configuration.Culture); } else { throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec))); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } } if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue)) { return(false); } } catch (ChoParserException) { Reader.IsValid = false; throw; } catch (ChoMissingRecordFieldException) { Reader.IsValid = false; if (Configuration.ThrowAndStopOnMissingField) { throw; } } catch (Exception ex) { Reader.IsValid = false; ChoETLFramework.HandleException(ref ex); if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop) { throw; } try { if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject) { var dict = rec as IDictionary <string, Object>; if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (ex is ValidationException) { throw; } else { throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } else if (pi != null) { if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (ex is ValidationException) { throw; } else { throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } else { throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } } catch (Exception innerEx) { if (ex == innerEx.InnerException || ex is ValidationException) { if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring field...".FormatString(ex.Message)); continue; } else { if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, ref fieldValue, ex)) { if (ex is ValidationException) { throw; } throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex); } else { try { if (Configuration.IsDynamicObject) { var dict = rec as IDictionary <string, Object>; dict.ConvertNSetMemberValue(kvp.Key, fieldConfig, ref fieldValue, Configuration.Culture); } else { if (pi != null) { rec.ConvertNSetMemberValue(kvp.Key, fieldConfig, ref fieldValue, Configuration.Culture); } else { throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec))); } } } catch { } } } } else { throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx); } } } } rec = rootRec; return(true); }
private bool ToText(long index, object rec, out string recText) { if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType)) { rec = ChoActivator.CreateInstance(Configuration.RecordType, rec); } if (!Configuration.IsDynamicObject) { if (rec.ToTextIfCustomSerialization(out recText)) { return(true); } //Check if KVP object if (rec.GetType().IsKeyValueType()) { recText = SerializeObject(rec); return(true); } } recText = null; if (Configuration.UseYamlSerialization) { recText = new Serializer().Serialize(rec, rec.GetType()); return(true); } StringBuilder msg = new StringBuilder(); object fieldValue = null; string fieldText = null; ChoYamlRecordFieldConfiguration fieldConfig = null; string fieldName = null; if (Configuration.ColumnCountStrict) { CheckColumnsStrict(rec); } //bool firstColumn = true; PropertyInfo pi = null; bool isFirst = true; object rootRec = rec; Dictionary <string, object> output = new Dictionary <string, object>(); foreach (KeyValuePair <string, ChoYamlRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict.OrderBy(kvp => kvp.Value.Order)) { //if (Configuration.IsDynamicObject) //{ if (Configuration.IgnoredFields.Contains(kvp.Key)) { continue; } //} fieldConfig = kvp.Value; fieldName = fieldConfig.FieldName; fieldValue = null; fieldText = String.Empty; if (Configuration.PIDict != null) { Configuration.PIDict.TryGetValue(kvp.Key, out pi); } rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec); if (Configuration.ThrowAndStopOnMissingField) { if (Configuration.IsDynamicObject) { var dict = rec.ToDynamicObject() as IDictionary <string, Object>; if (!dict.ContainsKey(kvp.Key)) { throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' Yaml node.".FormatString(fieldConfig.FieldName)); } } else { if (pi == null) { if (!RecordType.IsSimple()) { throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' Yaml node.".FormatString(fieldConfig.FieldName)); } } } } try { if (Configuration.IsDynamicObject) { IDictionary <string, Object> dict = rec.ToDynamicObject() as IDictionary <string, Object>; fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture); if (rec is ChoDynamicObject) { } if (kvp.Value.FieldType == null) { if (rec is ChoDynamicObject) { var dobj = rec as ChoDynamicObject; kvp.Value.FieldType = dobj.GetMemberType(kvp.Key); } if (kvp.Value.FieldType == null) { if (ElementType == null) { kvp.Value.FieldType = typeof(object); } else { kvp.Value.FieldType = ElementType; } } } } else { if (pi != null) { fieldValue = ChoType.GetPropertyValue(rec, pi); if (kvp.Value.FieldType == null) { kvp.Value.FieldType = pi.PropertyType; } } else { kvp.Value.FieldType = typeof(string); } } //Discover default value, use it if null //if (fieldValue == null) //{ // if (fieldConfig.IsDefaultValueSpecified) // fieldValue = fieldConfig.DefaultValue; //} bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode); if (ignoreFieldValue) { fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null; } if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue)) { return(false); } if (fieldConfig.ValueConverter != null) { fieldValue = fieldConfig.ValueConverter(fieldValue); } else if (RecordType.IsSimple()) { fieldValue = new List <object> { rec } } ; else { rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue, true); } if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); } if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue)) { return(false); } } catch (ChoParserException) { throw; } catch (ChoMissingRecordFieldException) { if (Configuration.ThrowAndStopOnMissingField) { throw; } } catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop) { throw; } try { if (Configuration.IsDynamicObject) { var dict = rec.ToDynamicObject() as IDictionary <string, Object>; if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); } else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); } else { var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex); fieldValue = null; throw ex1; } } else if (pi != null) { if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode); } else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue)) { rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue); } else { var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex); fieldValue = null; throw ex1; } } else { var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex); fieldValue = null; throw ex1; } } catch (Exception innerEx) { if (ex == innerEx.InnerException) { if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue) { continue; } else { if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, ref fieldValue, ex)) { throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex); } } } else { throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx); } } } bool isSimple = true; if (fieldConfig.CustomSerializer != null) { fieldText = fieldConfig.CustomSerializer(fieldValue) as string; } else if (RaiseRecordFieldSerialize(rec, index, kvp.Key, ref fieldValue)) { fieldText = fieldValue as string; } else if (fieldConfig.PropCustomSerializer != null) { fieldText = ChoCustomSerializer.Serialize(fieldValue, typeof(string), fieldConfig.PropCustomSerializer, fieldConfig.PropCustomSerializerParams, Configuration.Culture, fieldConfig.Name) as string; } else { Type ft = fieldValue == null ? typeof(object) : fieldValue.GetType(); //if (fieldConfig.IgnoreFieldValue(fieldValue)) // fieldText = null; bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode); if (ignoreFieldValue) { fieldText = null; } else if (fieldValue == null) { //if (fieldConfig.FieldType == null || fieldConfig.FieldType == typeof(object)) //{ // if (fieldConfig.NullValue == null) // fieldText = !fieldConfig.IsArray ? "null" : "[]"; // else // fieldText = fieldConfig.NullValue; //} if (Configuration.NullValueHandling == ChoNullValueHandling.Ignore) { fieldText = null; } else if (Configuration.NullValueHandling == ChoNullValueHandling.Default) { fieldText = ChoActivator.CreateInstance(fieldConfig.FieldType).ToNString(); } else if (Configuration.NullValueHandling == ChoNullValueHandling.Empty && fieldConfig.FieldType == typeof(string)) { fieldText = String.Empty; } else { if (fieldConfig.NullValue == null) { } else { fieldText = fieldConfig.NullValue; } } } else if (ft == typeof(string) || ft == typeof(char)) { fieldText = NormalizeFieldValue(kvp.Key, fieldValue.ToString(), kvp.Value.Size, kvp.Value.Truncate, false, GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType), GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false, kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType, Configuration.FieldValueTrimOption)); } else if (ft == typeof(DateTime) || ft == typeof(TimeSpan)) { fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string; } else if (ft.IsEnum) { fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string; } else if (ft == typeof(ChoCurrency)) { fieldText = fieldValue.ToString(); } else if (ft == typeof(bool)) { fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string; } else if (ft.IsNumeric()) { fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string; } else { isSimple = false; } } object objValue = null; if (isSimple) { objValue = fieldText; } else { Writer.ContractResolverState = new ChoContractResolverState { Name = kvp.Key, Index = index, Record = rec, FieldConfig = kvp.Value }; var json = JsonConvert.SerializeObject(fieldValue, Configuration.JsonSerializerSettings); if (RecordType.IsSimple()) { objValue = JsonConvert.DeserializeObject <IList <object> >(json); } else { objValue = JsonConvert.DeserializeObject <IDictionary <string, object> >(json); } } if (!RecordType.IsSimple()) { output.Add(fieldName, objValue); } else { fieldValue = objValue; } } if (!RecordType.IsSimple()) { recText = Configuration.YamlSerializer.Serialize(output, output.GetType()); } else { recText = Configuration.YamlSerializer.Serialize(fieldValue, fieldValue.GetType()); } return(true); }