private bool LoadNode(Tuple <long, IDictionary <string, object> > pair, ref object rec)
{
bool ignoreFieldValue = pair.Item2.IgnoreFieldValue(Configuration.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
return(false);
}
else if (pair.Item2 == null && !Configuration.IsDynamicObject)
{
rec = RecordType.CreateInstanceAndDefaultToMembers(Configuration.RecordFieldConfigurationsDict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value as ChoRecordFieldConfiguration));
return(true);
}
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordTypeSelector != null)
{
Type recType = Configuration.RecordTypeSelector(pair);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordTypeFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
return(true);
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
if (!Configuration.RecordTypeMapped)
{
Configuration.MapRecordFields(recType);
Configuration.Validate(null);
}
rec = recType.IsDynamicType() ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = Configuration.ThrowExceptionIfDynamicPropNotExists == null ? ChoDynamicObjectSettings.ThrowExceptionIfPropNotExists : Configuration.ThrowExceptionIfDynamicPropNotExists.Value,
} : ChoActivator.CreateInstance(recType);
RecordType = recType;
}
else if (Configuration.IsDynamicObject)
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = Configuration.ThrowExceptionIfDynamicPropNotExists == null ? ChoDynamicObjectSettings.ThrowExceptionIfPropNotExists : Configuration.ThrowExceptionIfDynamicPropNotExists.Value,
}
}
: ChoActivator.CreateInstance(RecordType);
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
//if (Configuration.CustomNodeSelecter != null)
//{
// pair = new Tuple<long, IDictionary<string, object>>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
//}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
if (!FillRecord(ref rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.ParquetRecordFieldConfigurations);
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
throw;
}
}
return(true);
}
return(true);
}
object fieldValue = null;
ChoParquetRecordFieldConfiguration fieldConfig = null;
PropertyInfo pi = null;
private bool LoadNode(Tuple <long, object> pair, ref object rec)
{
bool ignoreFieldValue = pair.Item2.IgnoreFieldValue(Configuration.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
return(false);
}
else if (pair.Item2 == null && !Configuration.IsDynamicObject)
{
rec = RecordType.CreateInstanceAndDefaultToMembers(Configuration.RecordFieldConfigurationsDict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value as ChoRecordFieldConfiguration));
return(true);
}
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
//if (Configuration.CustomNodeSelecter != null)
//{
// pair = new Tuple<long, object>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
//}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
rec = pair.Item2;
if (rec is AvroRecord)
{
rec = ToDynamicObject(rec as AvroRecord);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations);
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
throw;
}
}
return(true);
}
return(true);
}
