private bool FillRecord(object rec, Tuple <long, JObject> pair)
{
long lineNo;
JObject node;
JToken jToken = null;
lineNo = pair.Item1;
node = pair.Item2;
fJObjects = null;
fieldValue = null;
fieldConfig = null;
pi = null;
//IDictionary<string, object> dictValues = ToDictionary(node);
foreach (KeyValuePair <string, ChoJSONRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
//fieldValue = dictValues[kvp.Key];
if (!kvp.Value.JSONPath.IsNullOrWhiteSpace())
{
jToken = node.SelectToken(kvp.Value.JSONPath);
if (jToken == null)
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
}
else
{
if (!node.TryGetValue(kvp.Key, StringComparison.CurrentCultureIgnoreCase, out jToken))
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
}
if (rec is ExpandoObject)
{
if (kvp.Value.FieldType == null)
{
fieldValue = jToken;
}
else
{
fieldValue = jToken != null?jToken.ToObject(kvp.Value.FieldType) : null;
}
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
fieldValue = jToken != null?jToken.ToObject(kvp.Value.FieldType) : null;
}
if (!(fieldValue is ICollection))
{
if (fieldValue is string)
{
fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string);
}
}
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
private bool FillRecord(object rec, Tuple <long, JObject> pair)
{
long lineNo;
JObject node;
JToken jToken = null;
JToken[] jTokens = null;
lineNo = pair.Item1;
node = pair.Item2;
fieldValue = null;
fieldConfig = null;
pi = null;
//IDictionary<string, object> dictValues = ToDictionary(node);
if (!Configuration.IsDynamicObject)
{
if (rec.FillIfCustomSerialization(pair.Item2))
{
return(true);
}
if (FillIfKeyValueObject(rec, pair))
{
return(true);
}
}
foreach (KeyValuePair <string, ChoJSONRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
//fieldValue = dictValues[kvp.Key];
if (!kvp.Value.JSONPath.IsNullOrWhiteSpace())
{
jTokens = node.SelectTokens(kvp.Value.JSONPath).ToArray();
jToken = jTokens.FirstOrDefault();
if (jToken == null)
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
}
else
{
if (!node.TryGetValue(kvp.Key, StringComparison.CurrentCultureIgnoreCase, out jToken))
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
}
fieldValue = !jTokens.IsNullOrEmpty() ? (object)jTokens : jToken;
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
if (Configuration.IsDynamicObject) //rec is ExpandoObject)
{
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else
{
if (fieldConfig.FieldType == null)
{
if (!fieldConfig.IsArray && fieldValue is JToken[])
{
fieldValue = ((JToken[])fieldValue).FirstOrDefault();
if (fieldValue is JArray)
{
fieldValue = ((JArray)fieldValue).FirstOrDefault();
}
}
}
else
{
if (!fieldConfig.FieldType.IsCollection() && fieldValue is JToken[])
{
fieldValue = ((JToken[])fieldValue).FirstOrDefault();
//if (fieldValue is JArray)
//{
// fieldValue = ((JArray)fieldValue).FirstOrDefault();
//}
}
}
if (fieldConfig.FieldType == null ||
fieldConfig.FieldType == typeof(object) ||
fieldConfig.FieldType.GetItemType() == typeof(object))
{
if (fieldValue is JToken)
{
if (fieldConfig.ItemConverter != null)
{
fieldValue = fieldConfig.ItemConverter(fieldValue);
}
else
{
fieldValue = ToObject((JToken)fieldValue, null);
}
}
else if (fieldValue is JToken[])
{
List <object> arr = new List <object>();
foreach (var ele in (JToken[])fieldValue)
{
if (fieldConfig.ItemConverter != null)
{
arr.Add(fieldConfig.ItemConverter(ele));
}
else
{
arr.Add(ToObject(ele, null));
}
}
fieldValue = arr.ToArray();
}
}
else if (fieldConfig.FieldType == typeof(string) || fieldConfig.FieldType.IsSimple())
{
if (fieldValue is JToken[])
{
fieldValue = ((JToken[])fieldValue).FirstOrDefault();
}
if (fieldValue is JToken)
{
if (fieldConfig.ItemConverter != null)
{
fieldValue = fieldConfig.ItemConverter(fieldValue);
}
else
{
fieldValue = ToObject((JToken)fieldValue, fieldConfig.FieldType);
}
}
}
//else if (fieldConfig.FieldType.IsCollection())
//{
// List<object> list = new List<object>();
// Type itemType = fieldConfig.FieldType.GetItemType().GetUnderlyingType();
// if (fieldValue is JToken)
// {
// if (fieldConfig.ItemConverter != null)
// fieldValue = fieldConfig.ItemConverter(fieldValue);
// else
// fieldValue = ToObject((JToken)fieldValue, itemType);
// }
// else if (fieldValue is JToken[])
// {
// foreach (var ele in (JToken[])fieldValue)
// {
// if (fieldConfig.ItemConverter != null)
// list.Add(fieldConfig.ItemConverter(ele));
// else
// {
// fieldValue = ToObject(ele, itemType);
// }
// }
// fieldValue = list.ToArray();
// }
//}
else
{
List <object> list = new List <object>();
Type itemType = fieldConfig.FieldType.GetUnderlyingType().GetItemType().GetUnderlyingType();
if (fieldValue is JToken)
{
if (fieldConfig.ItemConverter != null)
{
fieldValue = fieldConfig.ItemConverter(fieldValue);
}
else
{
fieldValue = ToObject((JToken)fieldValue, itemType);
}
}
else if (fieldValue is JToken[])
{
var fi = ((JToken[])fieldValue).FirstOrDefault();
if (fi is JArray && !itemType.IsCollection())
{
if (fieldConfig.ItemConverter != null)
{
fieldValue = fieldConfig.ItemConverter(fi);
}
else
{
fieldValue = ToObject(fi, fieldConfig.FieldType);
}
}
else
{
foreach (var ele in (JToken[])fieldValue)
{
if (fieldConfig.ItemConverter != null)
{
list.Add(fieldConfig.ItemConverter(ele));
}
else
{
list.Add(ToObject(ele, itemType));
}
}
fieldValue = list.ToArray();
}
}
}
}
if (!(fieldValue is ICollection))
{
if (fieldValue is string)
{
fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string);
}
else if (fieldValue is JValue)
{
if (((JValue)fieldValue).Value is string)
{
fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue.ToString());
}
else
{
fieldValue = ((JValue)fieldValue).Value;
}
}
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
Reader.IsValid = false;
throw;
}
catch (ChoMissingRecordFieldException)
{
Reader.IsValid = false;
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}