private bool FillRecord(ref object rec, Tuple <long, IDictionary <string, object> > pair)
{
long lineNo;
lineNo = pair.Item1;
var node = pair.Item2;
fieldValue = null;
fieldConfig = null;
pi = null;
object rootRec = rec;
foreach (KeyValuePair <string, ChoParquetRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (Configuration.IgnoredFields.Contains(kvp.Key))
{
continue;
}
}
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
if (!node.ContainsKey(kvp.Value.FieldName))
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
fieldValue = node[kvp.Value.FieldName];
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
try
{
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
object v1 = node;
if (fieldConfig.CustomSerializer != null)
{
fieldValue = fieldConfig.CustomSerializer(v1);
}
else if (RaiseRecordFieldDeserialize(rec, pair.Item1, kvp.Key, ref v1))
{
fieldValue = v1;
}
else if (fieldConfig.PropCustomSerializer != null)
{
fieldValue = ChoCustomSerializer.Deserialize(v1, fieldConfig.FieldType, fieldConfig.PropCustomSerializer, fieldConfig.PropCustomSerializerParams, Configuration.Culture, fieldConfig.Name);
}
else
{
if (!node.ContainsKey(kvp.Value.FieldName))
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
else
{
fieldValue = node[kvp.Value.FieldName];
}
}
bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (Configuration.SupportsMultiRecordTypes)
{
ChoType.TryGetProperty(rec.GetType(), kvp.Key, out pi);
//*** TODO
//fieldConfig.PI = pi;
//fieldConfig.PropConverters = ChoTypeDescriptor.GetTypeConverters(fieldConfig.PI);
//fieldConfig.PropConverterParams = ChoTypeDescriptor.GetTypeConverterParams(fieldConfig.PI);
}
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else if (RecordType.IsSimple())
{
rec = ChoConvert.ConvertTo(fieldValue, RecordType, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
Reader.IsValid = false;
throw;
}
catch (ChoMissingRecordFieldException)
{
Reader.IsValid = false;
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException || ex is ValidationException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, ref fieldValue, ex))
{
if (ex is ValidationException)
{
throw;
}
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
else
{
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, fieldConfig, ref fieldValue, Configuration.Culture);
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, fieldConfig, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
}
}
catch { }
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
rec = rootRec;
return(true);
}
private bool ToText(long index, object rec, out string recText)
{
if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType))
{
rec = ChoActivator.CreateInstance(Configuration.RecordType, rec);
}
if (!Configuration.IsDynamicObject)
{
if (rec.ToTextIfCustomSerialization(out recText))
{
return(true);
}
//Check if KVP object
if (rec.GetType().IsKeyValueType())
{
recText = SerializeObject(rec);
return(true);
}
}
recText = null;
if (Configuration.UseYamlSerialization)
{
recText = Configuration.YamlSerializer.Serialize(rec);
return(true);
}
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
ChoYamlRecordFieldConfiguration fieldConfig = null;
string fieldName = null;
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
//bool firstColumn = true;
PropertyInfo pi = null;
bool isFirst = true;
object rootRec = rec;
Dictionary <string, object> output = new Dictionary <string, object>();
foreach (KeyValuePair <string, ChoYamlRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict.OrderBy(kvp => kvp.Value.Order))
{
//if (Configuration.IsDynamicObject)
//{
if (Configuration.IgnoredFields.Contains(kvp.Key))
{
continue;
}
//}
fieldConfig = kvp.Value;
fieldName = fieldConfig.FieldName;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
// if FieldName is set
if (!string.IsNullOrEmpty(fieldConfig.FieldName))
{
// match using FieldName
Configuration.PIDict.TryGetValue(fieldConfig.FieldName, out pi);
}
else
{
// otherwise match usign the property name
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
var dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (!dict.ContainsKey(kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' Yaml node.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (pi == null)
{
if (!RecordType.IsSimple())
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' Yaml node.".FormatString(fieldConfig.FieldName));
}
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
IDictionary <string, Object> dict = rec.ToDynamicObject() as IDictionary <string, Object>;
fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (rec is ChoDynamicObject)
{
}
if (kvp.Value.FieldType == null)
{
if (rec is ChoDynamicObject)
{
var dobj = rec as ChoDynamicObject;
kvp.Value.FieldType = dobj.GetMemberType(kvp.Key);
}
if (kvp.Value.FieldType == null)
{
if (ElementType == null)
{
kvp.Value.FieldType = typeof(object);
}
else
{
kvp.Value.FieldType = ElementType;
}
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
//if (fieldValue == null)
//{
// if (fieldConfig.IsDefaultValueSpecified)
// fieldValue = fieldConfig.DefaultValue;
//}
bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else if (RecordType.IsSimple())
{
fieldValue = new List <object> {
rec
}
}
;
else
{
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue, true);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, ref fieldValue, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
bool isSimple = true;
Type ft = fieldValue == null ? typeof(object) : fieldValue.GetType();
if (fieldConfig.CustomSerializer != null)
{
fieldText = fieldConfig.CustomSerializer(fieldValue) as string;
}
else if (RaiseRecordFieldSerialize(rec, index, kvp.Key, ref fieldValue))
{
fieldText = fieldValue as string;
}
else if (fieldConfig.PropCustomSerializer != null)
{
fieldText = ChoCustomSerializer.Serialize(fieldValue, typeof(string), fieldConfig.PropCustomSerializer, fieldConfig.PropCustomSerializerParams, Configuration.Culture, fieldConfig.Name) as string;
}
else
{
//if (fieldConfig.IgnoreFieldValue(fieldValue))
// fieldText = null;
bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
fieldText = null;
}
else if (fieldValue == null)
{
//if (fieldConfig.FieldType == null || fieldConfig.FieldType == typeof(object))
//{
// if (fieldConfig.NullValue == null)
// fieldText = !fieldConfig.IsArray ? "null" : "[]";
// else
// fieldText = fieldConfig.NullValue;
//}
if (Configuration.NullValueHandling == ChoNullValueHandling.Ignore)
{
fieldText = null;
}
else if (Configuration.NullValueHandling == ChoNullValueHandling.Default)
{
fieldText = ChoActivator.CreateInstance(fieldConfig.FieldType).ToNString();
}
else if (Configuration.NullValueHandling == ChoNullValueHandling.Empty && fieldConfig.FieldType == typeof(string))
{
fieldText = String.Empty;
}
else
{
if (fieldConfig.NullValue == null)
{
}
else
{
fieldText = fieldConfig.NullValue;
}
}
}
else if (ft == typeof(string) || ft == typeof(char))
{
fieldText = NormalizeFieldValue(kvp.Key, fieldValue.ToString(), kvp.Value.Size, kvp.Value.Truncate, false, GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType),
GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false, kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType, Configuration.FieldValueTrimOption));
}
else if (ft == typeof(DateTime) || ft == typeof(TimeSpan))
{
fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string;
}
else if (ft.IsEnum)
{
fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string;
}
else if (ft == typeof(ChoCurrency))
{
fieldText = fieldValue.ToString();
}
else if (ft == typeof(bool))
{
fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string;
}
else if (ft.IsNumeric())
{
fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string;
}
else
{
isSimple = false;
}
}
object objValue = null;
if (isSimple)
{
objValue = fieldText;
}
else
{
Writer.ContractResolverState = new ChoContractResolverState
{
Name = kvp.Key,
Index = index,
Record = rec,
FieldConfig = kvp.Value
};
var json = JsonConvert.SerializeObject(fieldValue, Configuration.JsonSerializerSettings);
if (RecordType.IsSimple())
{
objValue = JsonConvert.DeserializeObject <IList <object> >(json);
}
else if (typeof(IList).IsAssignableFrom(ft))
{
objValue = JsonConvert.DeserializeObject <IList>(json);
}
else
{
objValue = JsonConvert.DeserializeObject <IDictionary <string, object> >(json);
}
}
if (!RecordType.IsSimple())
{
output.Add(fieldName, objValue);
}
else
{
fieldValue = objValue;
}
}
if (!RecordType.IsSimple())
{
recText = Configuration.YamlSerializer.Serialize(output, output.GetType());
}
else
{
recText = Configuration.YamlSerializer.Serialize(fieldValue, fieldValue.GetType());
}
return(true);
}
