public ChoParquetReader <T> WithFields(params string[] fieldsNames)
{
string fnTrim = null;
if (!fieldsNames.IsNullOrEmpty())
{
int maxFieldPos = Configuration.ParquetRecordFieldConfigurations.Count > 0 ? Configuration.ParquetRecordFieldConfigurations.Max(f => f.FieldPosition) : 0;
PropertyDescriptor pd = null;
ChoParquetRecordFieldConfiguration fc = null;
foreach (string fn in fieldsNames)
{
if (fn.IsNullOrEmpty())
{
continue;
}
if (!_clearFields)
{
ClearFields();
Configuration.MapRecordFields(Configuration.RecordType);
//Configuration.ColumnOrderStrict = true;
}
fnTrim = fn.NTrim();
if (Configuration.ParquetRecordFieldConfigurations.Any(o => o.Name == fnTrim))
{
fc = Configuration.ParquetRecordFieldConfigurations.Where(o => o.Name == fnTrim).First();
Configuration.ParquetRecordFieldConfigurations.Remove(Configuration.ParquetRecordFieldConfigurations.Where(o => o.Name == fnTrim).First());
}
else
{
pd = ChoTypeDescriptor.GetProperty(typeof(T), fn);
}
var nfc = new ChoParquetRecordFieldConfiguration(fnTrim, ++maxFieldPos)
{
FieldName = fn
};
nfc.PropertyDescriptor = fc != null ? fc.PropertyDescriptor : pd;
nfc.DeclaringMember = fc != null ? fc.DeclaringMember : null;
if (pd != null)
{
if (nfc.FieldType == null)
{
nfc.FieldType = pd.PropertyType;
}
}
Configuration.ParquetRecordFieldConfigurations.Add(nfc);
}
}
return(this);
}
private bool LoadNode(Tuple <long, IDictionary <string, object> > pair, ref object rec)
{
bool ignoreFieldValue = pair.Item2.IgnoreFieldValue(Configuration.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
return(false);
}
else if (pair.Item2 == null && !Configuration.IsDynamicObject)
{
rec = RecordType.CreateInstanceAndDefaultToMembers(Configuration.RecordFieldConfigurationsDict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value as ChoRecordFieldConfiguration));
return(true);
}
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordTypeSelector != null)
{
Type recType = Configuration.RecordTypeSelector(pair);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordTypeFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
return(true);
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
if (!Configuration.RecordTypeMapped)
{
Configuration.MapRecordFields(recType);
Configuration.Validate(null);
}
rec = recType.IsDynamicType() ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = Configuration.ThrowExceptionIfDynamicPropNotExists == null ? ChoDynamicObjectSettings.ThrowExceptionIfPropNotExists : Configuration.ThrowExceptionIfDynamicPropNotExists.Value,
} : ChoActivator.CreateInstance(recType);
RecordType = recType;
}
else if (Configuration.IsDynamicObject)
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = Configuration.ThrowExceptionIfDynamicPropNotExists == null ? ChoDynamicObjectSettings.ThrowExceptionIfPropNotExists : Configuration.ThrowExceptionIfDynamicPropNotExists.Value,
}
}
: ChoActivator.CreateInstance(RecordType);
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
//if (Configuration.CustomNodeSelecter != null)
//{
// pair = new Tuple<long, IDictionary<string, object>>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
//}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
if (!FillRecord(ref rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.ParquetRecordFieldConfigurations);
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
throw;
}
}
return(true);
}
return(true);
}
object fieldValue = null;
ChoParquetRecordFieldConfiguration fieldConfig = null;
PropertyInfo pi = null;
internal ChoParquetRecordFieldConfigurationMap(ChoParquetRecordFieldConfiguration config)
{
ChoGuard.ArgumentNotNull(config, nameof(config));
_config = config;
}