Ejemplo n.º 1
0
        public ChoParquetReader <T> WithFields(params string[] fieldsNames)
        {
            string fnTrim = null;

            if (!fieldsNames.IsNullOrEmpty())
            {
                int maxFieldPos       = Configuration.ParquetRecordFieldConfigurations.Count > 0 ? Configuration.ParquetRecordFieldConfigurations.Max(f => f.FieldPosition) : 0;
                PropertyDescriptor pd = null;
                ChoParquetRecordFieldConfiguration fc = null;
                foreach (string fn in fieldsNames)
                {
                    if (fn.IsNullOrEmpty())
                    {
                        continue;
                    }
                    if (!_clearFields)
                    {
                        ClearFields();
                        Configuration.MapRecordFields(Configuration.RecordType);
                        //Configuration.ColumnOrderStrict = true;
                    }

                    fnTrim = fn.NTrim();
                    if (Configuration.ParquetRecordFieldConfigurations.Any(o => o.Name == fnTrim))
                    {
                        fc = Configuration.ParquetRecordFieldConfigurations.Where(o => o.Name == fnTrim).First();
                        Configuration.ParquetRecordFieldConfigurations.Remove(Configuration.ParquetRecordFieldConfigurations.Where(o => o.Name == fnTrim).First());
                    }
                    else
                    {
                        pd = ChoTypeDescriptor.GetProperty(typeof(T), fn);
                    }

                    var nfc = new ChoParquetRecordFieldConfiguration(fnTrim, ++maxFieldPos)
                    {
                        FieldName = fn
                    };
                    nfc.PropertyDescriptor = fc != null ? fc.PropertyDescriptor : pd;
                    nfc.DeclaringMember    = fc != null ? fc.DeclaringMember : null;
                    if (pd != null)
                    {
                        if (nfc.FieldType == null)
                        {
                            nfc.FieldType = pd.PropertyType;
                        }
                    }

                    Configuration.ParquetRecordFieldConfigurations.Add(nfc);
                }
            }

            return(this);
        }
Ejemplo n.º 2
0
        private bool LoadNode(Tuple <long, IDictionary <string, object> > pair, ref object rec)
        {
            bool ignoreFieldValue = pair.Item2.IgnoreFieldValue(Configuration.IgnoreFieldValueMode);

            if (ignoreFieldValue)
            {
                return(false);
            }
            else if (pair.Item2 == null && !Configuration.IsDynamicObject)
            {
                rec = RecordType.CreateInstanceAndDefaultToMembers(Configuration.RecordFieldConfigurationsDict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value as ChoRecordFieldConfiguration));
                return(true);
            }

            if (Configuration.SupportsMultiRecordTypes && Configuration.RecordTypeSelector != null)
            {
                Type recType = Configuration.RecordTypeSelector(pair);
                if (recType == null)
                {
                    if (Configuration.IgnoreIfNoRecordTypeFound)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
                        return(true);
                    }
                    else
                    {
                        throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
                    }
                }

                if (!Configuration.RecordTypeMapped)
                {
                    Configuration.MapRecordFields(recType);
                    Configuration.Validate(null);
                }

                rec = recType.IsDynamicType() ? new ChoDynamicObject()
                {
                    ThrowExceptionIfPropNotExists = Configuration.ThrowExceptionIfDynamicPropNotExists == null ? ChoDynamicObjectSettings.ThrowExceptionIfPropNotExists : Configuration.ThrowExceptionIfDynamicPropNotExists.Value,
                } : ChoActivator.CreateInstance(recType);
                RecordType = recType;
            }
            else if (Configuration.IsDynamicObject)
            {
                rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
                {
                    ThrowExceptionIfPropNotExists = Configuration.ThrowExceptionIfDynamicPropNotExists == null ? ChoDynamicObjectSettings.ThrowExceptionIfPropNotExists : Configuration.ThrowExceptionIfDynamicPropNotExists.Value,
                }
            }
            : ChoActivator.CreateInstance(RecordType);

            try
            {
                if (!RaiseBeforeRecordLoad(rec, ref pair))
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
                    rec = null;
                    return(true);
                }
                //if (Configuration.CustomNodeSelecter != null)
                //{
                //    pair = new Tuple<long, IDictionary<string, object>>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
                //}

                if (pair.Item2 == null)
                {
                    rec = null;
                    return(true);
                }

                if (!FillRecord(ref rec, pair))
                {
                    return(false);
                }

                if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
                {
                    rec.DoObjectLevelValidation(Configuration, Configuration.ParquetRecordFieldConfigurations);
                }


                bool skip = false;
                if (!RaiseAfterRecordLoad(rec, pair, ref skip))
                {
                    return(false);
                }
                else if (skip)
                {
                    rec = null;
                    return(true);
                }
            }
            catch (Exception ex)
            {
                Reader.IsValid = false;
                if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
                {
                    if (!RaiseRecordLoadError(rec, pair, ex))
                    {
                        throw;
                    }
                    else
                    {
                        //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                        //rec = null;
                    }
                }
                else
                {
                    ChoETLFramework.HandleException(ref ex);
                    if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                        rec = null;
                    }
                    else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
                    {
                        if (!RaiseRecordLoadError(rec, pair, ex))
                        {
                            throw;
                        }
                        else
                        {
                            //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                            //rec = null;
                        }
                    }
                    else
                    {
                        throw;
                    }
                }

                return(true);
            }

            return(true);
        }

        object fieldValue = null;
        ChoParquetRecordFieldConfiguration fieldConfig = null;
        PropertyInfo pi = null;
 internal ChoParquetRecordFieldConfigurationMap(ChoParquetRecordFieldConfiguration config)
 {
     ChoGuard.ArgumentNotNull(config, nameof(config));
     _config = config;
 }