Beispiel #1
0
        public static void Initialize(this object target)
        {
            if (target == null)
            {
                return;
            }

            object defaultValue = null;

            foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties <DefaultValueAttribute>(target.GetType()))
            {
                try
                {
                    defaultValue = ChoTypeDescriptor.GetPropetyAttribute <DefaultValueAttribute>(pd).Value;
                    if (defaultValue != null)
                    {
                        ChoType.ConvertNSetPropertyValue(target, pd.Name, defaultValue);
                    }
                }
                catch (Exception ex)
                {
                    ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceError, "Error while assigning default value '{0}' to '{1}' member. {2}".FormatString(defaultValue, ChoType.GetMemberName(pd), ex.Message));
                }
            }
            foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties <ChoDefaultValueAttribute>(target.GetType()))
            {
                if (ChoTypeDescriptor.GetPropetyAttribute <ChoDefaultValueAttribute>(pd) == null)
                {
                    continue;
                }

                try
                {
                    defaultValue = ChoTypeDescriptor.GetPropetyAttribute <ChoDefaultValueAttribute>(pd).Value;
                    if (defaultValue != null)
                    {
                        ChoType.ConvertNSetPropertyValue(target, pd.Name, defaultValue);
                    }
                }
                catch (Exception ex)
                {
                    ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceError, "Error while assigning default value '{0}' to '{1}' member. {2}".FormatString(defaultValue, ChoType.GetMemberName(pd), ex.Message));
                }
            }

            //ChoETLFramework.InitializeObject(target);

            if (target is IChoInitializable)
            {
                ((IChoInitializable)target).Initialize();
            }
        }
Beispiel #2
0
        private void LoadFixedLengthRecordFieldOptions(ChoIniFile iniFile1)
        {
            ChoIniFile iniFile = iniFile1.GetSection("FIXED_LENGTH_RECORD_FIELD");

            var dict = new OrderedDictionary();
            ChoFixedLengthRecordFieldAttribute value;
            Dictionary <string, string>        iniKeyValuesDict = iniFile.KeyValuesDict;

            if (iniKeyValuesDict.Count == 0)
            {
                throw new ApplicationException("Missing fixed length field names in the '{0}' ini file.".FormatString(iniFile1.FilePath));
            }
            foreach (string fieldName in iniKeyValuesDict.Keys) //GetFieldNames(iniFile1))
            {
                if (fieldName.IsNullOrWhiteSpace())
                {
                    continue;
                }
                if (!iniKeyValuesDict.ContainsKey(fieldName))
                {
                    continue;
                }

                try
                {
                    value = new ChoFixedLengthRecordFieldAttribute();
                    value.FieldValueTrimOption    = ChoFieldValueTrimOption.Trim;
                    value.FieldValueJustification = ChoFieldValueJustification.Left;
                    foreach (KeyValuePair <string, string> kvp in iniKeyValuesDict[fieldName].ToKeyValuePairs())
                    {
                        try
                        {
                            ChoType.ConvertNSetMemberValue(value, kvp.Key, kvp.Value);
                        }
                        catch (Exception ex)
                        {
                            ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Error while loading record field option '{0}' for '{1}' field. {2}".FormatString(iniKeyValuesDict[fieldName], fieldName, ex.Message));
                        }
                    }
                    value.Validate();
                    dict.Add(fieldName, value);
                }
                catch (Exception ex)
                {
                    throw new ApplicationException("[FieldName: {0}] - {1}".FormatString(fieldName, ex.Message));
                }
            }
            _flRecordFieldOptionCache.Add(iniFile1.Key, dict);
        }
        private DateTime GetNowTime()
        {
            DateTime now = DateTime.Now;

            string nowText = ChoETLFramework.GetConfigValue("NOW");

            if (!nowText.IsNullOrWhiteSpace())
            {
                try
                {
                    now = Convert.ToDateTime(nowText);
                }
                catch { }
            }

            return(now);
        }
        private DateTime GetTodaysDate()
        {
            DateTime today = DateTime.Today;

            string todayText = ChoETLFramework.GetConfigValue("TODAY");

            if (!todayText.IsNullOrWhiteSpace())
            {
                try
                {
                    today = Convert.ToDateTime(todayText);
                }
                catch { }
            }

            return(today);
        }
Beispiel #5
0
        private bool ValidateOnWrite(ref object value)
        {
            var crs = Writer.ContractResolverState;
            var fc  = crs.FieldConfig;

            crs.Name = _fc == null?_mi.GetFullName() : crs.Name;

            var rec  = ChoType.GetMemberObjectMatchingType(crs.Name, crs.Record);
            var name = ChoType.GetFieldName(crs.Name);

            try
            {
                Validate(value);
            }
            catch (Exception ex)
            {
                ChoETLFramework.HandleException(ref ex);

                if (fc != null && fc.ErrorMode == ChoErrorMode.ThrowAndStop)
                {
                    throw;
                }

                if (fc != null)
                {
                    if (fc.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                    {
                        return(false);
                    }
                    else
                    {
                        if (!RaiseRecordFieldWriteError(rec, crs.Index, name, ref value, ex))
                        {
                            throw new ChoWriterException($"Failed to write '{value}' value of '{crs.Name}' member.", ex);
                        }
                    }
                }
                else
                {
                    throw new ChoWriterException($"Failed to write '{value}' value of '{crs.Name}' member.", ex);
                }
            }

            return(true);
        }
Beispiel #6
0
        private bool LoadExcelSeperatorIfAny(Tuple <int, string> pair)
        {
            string line = pair.Item2.NTrim();

            if (!line.IsNullOrWhiteSpace() && line.StartsWith("sep=", true, Configuration.Culture))
            {
                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Excel separator specified at [{0}]...".FormatString(pair.Item1));
                string delimiter = line.Substring(4);
                if (!delimiter.IsNullOrWhiteSpace())
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Excel separator [{0}] found.".FormatString(delimiter));
                    Configuration.Delimiter = delimiter;
                }

                return(true);
            }

            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Excel separator NOT found. Default separator [{0}] used.".FormatString(Configuration.Delimiter));
            return(false);
        }
        private void LoadConverterParams(Type declaringType)
        {
            ChoIniFile iniFile = GetIniSection(declaringType, "FORMATTER");

            var                dict            = new Dictionary <string, object[]>();
            string             parameters      = null;
            ChoMemberAttribute memberAttribute = null;

            foreach (MemberInfo memberInfo in ChoType.GetMembers(declaringType))
            {
                memberAttribute = ChoType.GetAttribute <ChoMemberAttribute>(memberInfo);
                if (memberAttribute == null)
                {
                    continue;
                }

                try
                {
                    if (_turnOnMetaDataCache)
                    {
                        parameters = iniFile.GetValue(memberInfo.Name);
                    }

                    List <object> p = new List <object>();
                    if (!parameters.IsNullOrWhiteSpace())
                    {
                        foreach (string kv in parameters.SplitNTrim(';'))
                        {
                            p.Add(kv.SplitNTrim(','));
                        }
                    }

                    dict.Add(memberInfo.Name, parameters.IsNullOrWhiteSpace() ? null : p.ToArray());
                }
                catch (Exception ex)
                {
                    ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Failed to retrieve converter params for '{0}' member type from INI file. {1}".FormatString(ChoType.GetMemberName(memberInfo), ex.Message));
                }
            }
            _converterParams.Add(declaringType, dict);
        }
        private void Start()
        {
            lock (typeof(ChoTextWriterTraceListener))
            {
                if (_fileWriterThread != null && _fileWriterThread.IsAlive)
                {
                    return;
                }

                try
                {
                    _fileWriterThread = new Thread(new ThreadStart(SyncFileWriter));
                    _fileWriterThread.IsBackground = true;
                    _fileWriterThread.Start();
                }
                catch (Exception ex)
                {
                    ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceError, ex.ToString());
                }
            }
        }
        private void LoadIsRequireds(Type declaringType)
        {
            ChoIniFile iniFile = GetIniSection(declaringType, "REQUIRED");

            var  dict       = new Dictionary <string, bool>();
            bool isRequired = false;
            ChoMemberAttribute memberAttribute = null;

            foreach (MemberInfo memberInfo in ChoType.GetMembers(declaringType))
            {
                memberAttribute = ChoType.GetAttribute <ChoMemberAttribute>(memberInfo);
                if (memberAttribute == null)
                {
                    continue;
                }

                try
                {
                    if (_turnOnMetaDataCache)
                    {
                        isRequired = iniFile.GetValue(memberInfo.Name, memberAttribute.IsRequired, _isLockedCache[declaringType]);
                    }
                    else
                    {
                        isRequired = memberAttribute.IsRequired;
                    }

                    dict.Add(memberInfo.Name, isRequired);
                }
                catch (Exception ex)
                {
                    ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Incorrect IsRequired value specified for '{0}' member type in INI file. Defaulted to false. {1}".FormatString(ChoType.GetMemberName(memberInfo), ex.Message));
                }
            }

            _isReqCache.Add(declaringType, dict);
        }
Beispiel #10
0
        private void LoadDbRecordFieldOptions(ChoIniFile iniFile1)
        {
            ChoIniFile iniFile = iniFile1.GetSection("Db_RECORD_FIELD");

            var dict = new Dictionary <string, ChoDbRecordFieldAttribute>();
            ChoDbRecordFieldAttribute   value;
            Dictionary <string, string> iniKeyValuesDict = iniFile.KeyValuesDict;

            foreach (string fieldName in iniKeyValuesDict.Keys)
            {
                if (fieldName.IsNullOrWhiteSpace())
                {
                    continue;
                }
                if (!iniKeyValuesDict.ContainsKey(fieldName))
                {
                    continue;
                }

                value = new ChoDbRecordFieldAttribute();
                foreach (KeyValuePair <string, string> kvp in iniKeyValuesDict[fieldName].ToKeyValuePairs())
                {
                    try
                    {
                        ChoType.ConvertNSetMemberValue(value, kvp.Key, kvp.Value);
                    }
                    catch (Exception ex)
                    {
                        ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Error while loading record field '{0}' for '{1}' field. {2}".FormatString(iniKeyValuesDict[fieldName], fieldName, ex.Message));
                    }
                }
                value.Validate();
                dict.Add(fieldName, value);
            }
            _dbRecordFieldOptionCache.Add(iniFile1.Key, dict);
        }
        private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
        {
            TraceSwitch = traceSwitch;

            TextReader sr = source as TextReader;

            ChoGuard.ArgumentNotNull(sr, "TextReader");

            if (sr is StreamReader)
            {
                ((StreamReader)sr).Seek(0, SeekOrigin.Begin);
            }

            if (Configuration.RecordSelector == null)
            {
                throw new ChoRecordConfigurationException("Missing record selector.");
            }

            if (!RaiseBeginLoad(sr))
            {
                yield break;
            }

            string[] commentTokens = Configuration.Comments;
            bool?    skip          = false;
            bool     _headerFound  = false;
            bool?    skipUntil     = true;
            bool?    doWhile       = true;

            using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
                       new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData, Configuration.MaxLineSize)).ToEnumerable(),
                       (pair) =>
            {
                //bool isStateAvail = IsStateAvail();

                skip = false;
                if (skipUntil != null)
                {
                    if (skipUntil.Value)
                    {
                        skipUntil = RaiseSkipUntil(pair);
                        if (skipUntil == null)
                        {
                        }
                        else
                        {
                            skip = skipUntil.Value;
                        }
                    }
                }

                //if (isStateAvail)
                //{
                //    if (!IsStateMatches(item))
                //    {
                //        skip = filterFunc != null ? filterFunc(item) : false;
                //    }
                //    else
                //        skip = true;
                //}
                //else
                //    skip = filterFunc != null ? filterFunc(item) : false;

                if (skip == null)
                {
                    return(null);
                }


                if (TraceSwitch.TraceVerbose)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);

                    if (!skip.Value)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
                    }
                    else
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
                    }
                }

                if (skip.Value)
                {
                    return(skip);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));

                //if (Task != null)
                //    return !IsStateNOTExistsOrNOTMatch(item);

                if (pair.Item2.IsNullOrWhiteSpace())
                {
                    if (!Configuration.IgnoreEmptyLine)
                    {
                        throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
                    }
                    else
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
                        }
                        return(true);
                    }
                }

                if (commentTokens != null && commentTokens.Length > 0)
                {
                    foreach (string comment in commentTokens)
                    {
                        if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal))     //, true, Configuration.Culture))
                        {
                            if (TraceSwitch.TraceVerbose)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
                            }
                            return(true);
                        }
                    }
                }

                if (!_configCheckDone)
                {
                    Configuration.Validate(pair);     // GetHeaders(pair.Item2));
                    _configCheckDone = true;
                }

                //Ignore Header if any
                if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
                    !_headerFound)
                {
                    if (TraceSwitch.TraceVerbose)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
                    }
                    _headerFound = true;
                    return(true);
                }

                return(false);
            }))
            {
                while (true)
                {
                    Tuple <long, string> pair = e.Peek;
                    if (pair == null)
                    {
                        RaiseEndLoad(sr);
                        yield break;
                    }

                    Type recType = Configuration.RecordSelector(pair.Item2);
                    if (recType == null)
                    {
                        if (Configuration.IgnoreIfNoRecordParserExists)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
                        }
                        else
                        {
                            throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
                        }
                    }

                    object rec = ChoActivator.CreateInstance(recType);
                    if (!LoadLine(pair, ref rec))
                    {
                        yield break;
                    }

                    //StoreState(e.Current, rec != null);

                    e.MoveNext();

                    if (rec == null)
                    {
                        continue;
                    }

                    yield return(rec);

                    if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
                    {
                        if (RaisedRowsLoaded(pair.Item1))
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
                            yield break;
                        }
                    }

                    if (doWhile != null)
                    {
                        doWhile = RaiseDoWhile(pair);
                        if (doWhile != null && doWhile.Value)
                        {
                            break;
                        }
                    }
                }
            }
        }
        private bool LoadLine(Tuple <long, string> pair, ref object rec)
        {
            Type recType = rec.GetType();

            try
            {
                if (!RaiseBeforeRecordLoad(rec, ref pair))
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
                    rec = null;
                    return(true);
                }

                if (pair.Item2 == null)
                {
                    rec = null;
                    return(true);
                }
                else if (pair.Item2 == String.Empty)
                {
                    return(true);
                }

                if (!pair.Item2.IsNullOrWhiteSpace())
                {
                    var config = GetConfiguration(rec.GetType());
                    if (config == null)
                    {
                        throw new ChoParserException("No parser found to parse record line.");
                    }

                    if (config.GetType() == typeof(ChoCSVRecordConfiguration))
                    {
                        var r = ChoCSVReader.LoadText(rec.GetType(), pair.Item2, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, TraceSwitch, Reader);
                        rec = r.FirstOrDefault <object>();
                    }
                    else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration))
                    {
                        var r = ChoFixedLengthReader.LoadText(rec.GetType(), pair.Item2, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, TraceSwitch, Reader);
                        rec = r.FirstOrDefault <object>();
                    }
                    else
                    {
                        throw new ChoParserException("Unsupported record line found to parse.");
                    }
                }

                bool skip = false;
                if (!RaiseAfterRecordLoad(rec, pair, ref skip))
                {
                    return(false);
                }
                else if (skip)
                {
                    rec = null;
                    return(true);
                }
            }
            catch (ChoParserException pEx)
            {
                throw new ChoParserException($"Failed to parse line to '{recType}' object.", pEx);
            }
            catch (ChoMissingRecordFieldException mEx)
            {
                throw new ChoParserException($"Failed to parse line to '{recType}' object.", mEx);
            }
            catch (Exception ex)
            {
                Reader.IsValid = false;
                ChoETLFramework.HandleException(ref ex);
                if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                    rec = null;
                }
                else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
                {
                    if (!RaiseRecordLoadError(rec, pair, ex))
                    {
                        throw;
                    }
                    else
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                        rec = null;
                    }
                }
                else
                {
                    throw;
                }
                //if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                //{
                //    rec = null;
                //}
                //else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
                //{
                //    if (!RaiseRecordLoadError(rec, pair, ex))
                //        throw new ChoReaderException($"Failed to parse line to '{recType}' object.", ex);
                //}
                //else
                //    throw new ChoReaderException($"Failed to parse line to '{recType}' object.", ex);

                return(true);
            }

            return(true);
        }
        private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
        {
            TraceSwitch = traceSwitch;

            TextReader sr = source as TextReader;

            ChoGuard.ArgumentNotNull(sr, "TextReader");

            if (sr is StreamReader)
            {
                ((StreamReader)sr).Seek(0, SeekOrigin.Begin);
            }

            if (!RaiseBeginLoad(sr))
            {
                yield break;
            }

            string[]                     commentTokens      = Configuration.Comments;
            bool?                        skip               = false;
            bool                         isRecordStartFound = false;
            bool                         isRecordEndFound   = false;
            long                         seekOriginPos      = sr is StreamReader ? ((StreamReader)sr).BaseStream.Position : 0;
            List <string>                headers            = new List <string>();
            Tuple <long, string>         lastLine           = null;
            List <Tuple <long, string> > recLines           = new List <Tuple <long, string> >();
            long                         recNo              = 0;
            int  loopCount      = Configuration.AutoDiscoverColumns && Configuration.KVPRecordFieldConfigurations.Count == 0 ? 2 : 1;
            bool isHeaderFound  = loopCount == 1;
            bool IsHeaderLoaded = false;
            Tuple <long, string> pairIn;
            bool abortRequested = false;

            for (int i = 0; i < loopCount; i++)
            {
                if (i == 1)
                {
                    if (sr is StreamReader)
                    {
                        ((StreamReader)sr).Seek(seekOriginPos, SeekOrigin.Begin);
                    }
                    TraceSwitch = traceSwitch;
                }
                else
                {
                    TraceSwitch = ChoETLFramework.TraceSwitchOff;
                }
                lastLine = null;
                recLines.Clear();
                isRecordEndFound   = false;
                isRecordStartFound = false;

                using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
                           new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(),
                           (pair) =>
                {
                    //bool isStateAvail = IsStateAvail();
                    skip = false;

                    //if (isStateAvail)
                    //{
                    //    if (!IsStateMatches(item))
                    //    {
                    //        skip = filterFunc != null ? filterFunc(item) : false;
                    //    }
                    //    else
                    //        skip = true;
                    //}
                    //else
                    //    skip = filterFunc != null ? filterFunc(item) : false;

                    if (skip == null)
                    {
                        return(null);
                    }


                    if (TraceSwitch.TraceVerbose)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);

                        if (!skip.Value)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
                        }
                        else
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
                        }
                    }

                    if (skip.Value)
                    {
                        return(skip);
                    }

                    //if (!(sr.BaseStream is MemoryStream))
                    //    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));

                    //if (Task != null)
                    //    return !IsStateNOTExistsOrNOTMatch(item);

                    if (pair.Item2.IsNullOrWhiteSpace())
                    {
                        if (!Configuration.IgnoreEmptyLine)
                        {
                            throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
                        }
                        else
                        {
                            if (TraceSwitch.TraceVerbose)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
                            }
                            return(true);
                        }
                    }

                    if (commentTokens != null && commentTokens.Length > 0)
                    {
                        foreach (string comment in commentTokens)
                        {
                            if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal))     //, true, Configuration.Culture))
                            {
                                if (TraceSwitch.TraceVerbose)
                                {
                                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
                                }
                                return(true);
                            }
                        }
                    }

                    if (!_configCheckDone)
                    {
                        Configuration.Validate(null);
                        var dict = Configuration.KVPRecordFieldConfigurations.ToDictionary(i1 => i1.Name, i1 => i1.FieldType == null ? null : i1.FieldType);
                        RaiseMembersDiscovered(dict);
                        Configuration.UpdateFieldTypesIfAny(dict);
                        _configCheckDone = true;
                    }

                    return(false);
                }))
                {
                    while (true)
                    {
                        pairIn = e.Peek;

                        if (!isRecordStartFound)
                        {
                            if (pairIn == null)
                            {
                                break;
                            }

                            lastLine = null;
                            recLines.Clear();
                            isRecordEndFound   = false;
                            isRecordStartFound = true;
                            if (!Configuration.RecordStart.IsNullOrWhiteSpace())
                            {
                                //Move to record start
                                while (!(Configuration.IsRecordStartMatch(e.Peek.Item2)))
                                {
                                    e.MoveNext();
                                    if (e.Peek == null)
                                    {
                                        break;
                                    }
                                }
                            }
                            if (e.Peek != null)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record start found at [{0}] line...".FormatString(e.Peek.Item1));
                            }

                            e.MoveNext();
                            continue;
                        }
                        else
                        {
                            string recordEnd = !Configuration.RecordEnd.IsNullOrWhiteSpace() ? Configuration.RecordEnd : Configuration.RecordStart;
                            if (!recordEnd.IsNullOrWhiteSpace())
                            {
                                if (e.Peek == null)
                                {
                                    if (Configuration.RecordEnd.IsNullOrWhiteSpace())
                                    {
                                        isRecordEndFound = true;
                                    }
                                    else
                                    {
                                        break;
                                    }
                                }
                                else
                                {
                                    //Move to record start
                                    if (Configuration.IsRecordEndMatch(e.Peek.Item2))
                                    {
                                        isRecordEndFound   = true;
                                        isRecordStartFound = false;
                                        if (e.Peek != null)
                                        {
                                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record end found at [{0}] line...".FormatString(e.Peek.Item1));
                                        }
                                    }
                                }
                            }
                            else if (e.Peek == null)
                            {
                                isRecordEndFound = true;
                            }

                            if (!isHeaderFound)
                            {
                                //if (isRecordEndFound && headers.Count == 0)
                                //{
                                //    //throw new ChoParserException("Unexpected EOF found.");
                                //}
                                if (!isRecordEndFound)
                                {
                                    e.MoveNext();
                                    if (e.Peek != null)
                                    {
                                        //If line empty or line continuation, skip
                                        if (pairIn.Item2.IsNullOrWhiteSpace() || IsLineContinuationCharFound(pairIn.Item2)) //.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
                                        {
                                        }
                                        else
                                        {
                                            string header = ToKVP(pairIn.Item1, pairIn.Item2).Key;
                                            if (!header.IsNullOrWhiteSpace())
                                            {
                                                headers.Add(header);
                                            }
                                        }
                                    }
                                }
                                else
                                {
                                    Configuration.Validate(headers.ToArray());
                                    isHeaderFound    = true;
                                    isRecordEndFound = false;
                                    IsHeaderLoaded   = true;
                                    break;
                                }
                            }
                            else
                            {
                                if (!IsHeaderLoaded)
                                {
                                    Configuration.Validate(new string[] { });
                                    IsHeaderLoaded = true;
                                }

                                if (isRecordEndFound && recLines.Count == 0)
                                {
                                    //throw new ChoParserException("Unexpected EOF found.");
                                }
                                else if (!isRecordEndFound)
                                {
                                    e.MoveNext();
                                    if (e.Peek != null)
                                    {
                                        //If line empty or line continuation, skip
                                        if (pairIn.Item2.IsNullOrWhiteSpace())
                                        {
                                            if (!Configuration.IgnoreEmptyLine)
                                            {
                                                throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pairIn.Item1));
                                            }
                                            else
                                            {
                                                Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter);
                                                recLines.RemoveAt(recLines.Count - 1);
                                                recLines.Add(t);
                                            }
                                        }
                                        else if (IsLineContinuationCharFound(pairIn.Item2)) //pairIn.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
                                        {
                                            if (lastLine == null)
                                            {
                                                throw new ChoParserException("Unexpected line continuation found at {0} location.".FormatString(pairIn.Item1));
                                            }
                                            else
                                            {
                                                Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter + pairIn.Item2);
                                                recLines.RemoveAt(recLines.Count - 1);
                                                recLines.Add(t);
                                            }
                                        }
                                        else
                                        {
                                            lastLine = pairIn;
                                            recLines.Add(pairIn);
                                        }
                                    }
                                }
                                else
                                {
                                    object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
                                    {
                                        ThrowExceptionIfPropNotExists = true,
                                        AlternativeKeys = Configuration.AlternativeKeys
                                    } : Activator.CreateInstance(RecordType);
                                    if (!LoadLines(new Tuple <long, List <Tuple <long, string> > >(++recNo, recLines), ref rec))
                                    {
                                        yield break;
                                    }

                                    isRecordStartFound = false;
                                    //StoreState(e.Current, rec != null);

                                    if (!Configuration.RecordEnd.IsNullOrWhiteSpace())
                                    {
                                        e.MoveNext();
                                    }

                                    if (rec == null)
                                    {
                                        continue;
                                    }

                                    yield return(rec);

                                    if (Configuration.NotifyAfter > 0 && recNo % Configuration.NotifyAfter == 0)
                                    {
                                        if (RaisedRowsLoaded(recNo))
                                        {
                                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
                                            abortRequested = true;
                                            yield break;
                                        }
                                    }

                                    if (e.Peek == null)
                                    {
                                        break;
                                    }
                                }
                            }
                        }
                    }
                }
            }

            if (!abortRequested)
            {
                RaisedRowsLoaded(recNo, true);
            }
            RaiseEndLoad(sr);
        }
Beispiel #14
0
        private bool ToText(long index, object rec, out string recText)
        {
            if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType))
            {
                rec = ChoActivator.CreateInstance(Configuration.RecordType, rec);
            }

            recText = null;
            StringBuilder msg        = new StringBuilder();
            object        fieldValue = null;
            string        fieldText  = null;
            ChoFixedLengthRecordFieldConfiguration fieldConfig = null;

            if (Configuration.ColumnCountStrict)
            {
                CheckColumnsStrict(rec);
            }

            //bool firstColumn = true;
            PropertyInfo pi      = null;
            object       rootRec = rec;
            IDictionary <string, Object> dict = null;

            foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
            {
                fieldConfig = kvp.Value;
                fieldValue  = null;
                fieldText   = String.Empty;
                if (Configuration.PIDict != null)
                {
                    Configuration.PIDict.TryGetValue(kvp.Key, out pi);
                }

                rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);

                dict = rec.ToDynamicObject() as IDictionary <string, Object>;
                if (Configuration.IsDynamicObject)
                {
                    dict = dict.Flatten().ToDictionary();
                }

                if (Configuration.ThrowAndStopOnMissingField)
                {
                    if (Configuration.IsDynamicObject)
                    {
                        if (!dict.ContainsKey(kvp.Key))
                        {
                            throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
                        }
                    }
                    else
                    {
                        if (pi == null)
                        {
                            throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
                        }
                    }
                }

                try
                {
                    if (Configuration.IsDynamicObject)
                    {
                        fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
                        if (kvp.Value.FieldType == null)
                        {
                            if (fieldValue == null)
                            {
                                kvp.Value.FieldType = typeof(string);
                            }
                            else
                            {
                                kvp.Value.FieldType = fieldValue.GetType();
                            }
                        }
                    }
                    else
                    {
                        if (pi != null)
                        {
                            fieldValue = ChoType.GetPropertyValue(rec, pi);
                            if (kvp.Value.FieldType == null)
                            {
                                kvp.Value.FieldType = pi.PropertyType;
                            }
                        }
                        else
                        {
                            kvp.Value.FieldType = typeof(string);
                        }
                    }

                    //Discover default value, use it if null
                    if (fieldValue == null)
                    {
                        if (fieldConfig.IsDefaultValueSpecified)
                        {
                            fieldValue = fieldConfig.DefaultValue;
                        }
                    }

                    if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
                    {
                        return(false);
                    }

                    if (fieldConfig.ValueConverter != null)
                    {
                        fieldValue = fieldConfig.ValueConverter(fieldValue);
                    }
                    else
                    {
                        rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue);
                    }

                    if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
                    {
                        rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
                    }

                    if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
                    {
                        return(false);
                    }
                }
                catch (ChoParserException)
                {
                    throw;
                }
                catch (ChoMissingRecordFieldException)
                {
                    if (Configuration.ThrowAndStopOnMissingField)
                    {
                        throw;
                    }
                }
                catch (Exception ex)
                {
                    ChoETLFramework.HandleException(ref ex);

                    if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
                    {
                        throw;
                    }

                    try
                    {
                        if (Configuration.IsDynamicObject)
                        {
                            if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
                            {
                                dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
                            }
                            else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
                            {
                                dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
                            }
                            else
                            {
                                var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
                                fieldValue = null;
                                throw ex1;
                            }
                        }
                        else if (pi != null)
                        {
                            if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
                            {
                                rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
                            }
                            else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
                            {
                                rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
                            }
                            else
                            {
                                var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
                                fieldValue = null;
                                throw ex1;
                            }
                        }
                        else
                        {
                            var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
                            fieldValue = null;
                            throw ex1;
                        }
                    }
                    catch (Exception innerEx)
                    {
                        if (ex == innerEx.InnerException)
                        {
                            if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                            {
                                continue;
                            }
                            else
                            {
                                if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
                                {
                                    throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
                                }
                            }
                        }
                        else
                        {
                            throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
                        }
                    }
                }

                if (fieldValue == null)
                {
                    fieldText = String.Empty;
                }
                else
                {
                    fieldText = fieldValue.ToString();
                }

                msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField,
                                               GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType),
                                               GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false, kvp.Value.NullValue,
                                               kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType)));
            }

            recText = msg.ToString();
            return(true);
        }
Beispiel #15
0
 static ChoRecordWriter()
 {
     ChoETLFramework.Initialize();
 }
Beispiel #16
0
 public static void Write(string msg)
 {
     ChoETLFramework.WriteLog(msg);
 }
Beispiel #17
0
 public static void Warning(string msg)
 {
     ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceWarning, msg);
 }
Beispiel #18
0
 public static void Verbose(string msg)
 {
     ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceVerbose, msg);
 }
Beispiel #19
0
 public static void Info(string msg)
 {
     ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceInfo, msg);
 }
        private void LoadDefaults(Type declaringType)
        {
            ChoIniFile iniFile = GetIniSection(declaringType, "DEFAULT_VALUE");

            var    dict         = new Dictionary <string, object>();
            object defaultValue = null;
            string memberName   = null;

            foreach (PropertyDescriptor pd in TypeDescriptor.GetProperties(declaringType).AsTypedEnumerable <PropertyDescriptor>().Where(pd => !pd.Attributes.OfType <ChoIgnoreMemberAttribute>().Any()))
            {
                try
                {
                    memberName = pd.Name;
                    if (_turnOnMetaDataCache)
                    {
                        defaultValue = iniFile.GetValue(pd.Name, ChoType.GetDefaultValue(pd), _isLockedCache[declaringType]);
                    }
                    else
                    {
                        defaultValue = ChoType.GetDefaultValue(pd);
                    }

                    if (defaultValue is string)
                    {
                        defaultValue = ((string)defaultValue).ExpandProperties();
                    }

                    defaultValue = ChoConvert.ConvertFrom(defaultValue, pd.PropertyType);

                    dict.Add(pd.Name, defaultValue);
                }
                catch (Exception ex)
                {
                    ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Error while converting default value '{0}' to '{1}' member type. {2}".FormatString(defaultValue, ChoType.GetMemberName(pd), ex.Message));
                }
            }

            foreach (FieldInfo memberInfo in ChoType.GetFields(declaringType))
            {
                try
                {
                    memberName = memberInfo.Name;
                    if (_turnOnMetaDataCache)
                    {
                        defaultValue = iniFile.GetValue(memberInfo.Name, ChoType.GetDefaultValue(memberInfo), _isLockedCache[declaringType]);
                    }
                    else
                    {
                        defaultValue = ChoType.GetDefaultValue(memberInfo);
                    }

                    if (defaultValue is string)
                    {
                        defaultValue = ((string)defaultValue).ExpandProperties();
                    }

                    defaultValue = ChoConvert.ConvertFrom(defaultValue, memberInfo.FieldType);

                    dict.Add(memberInfo.Name, defaultValue);
                }
                catch (Exception ex)
                {
                    ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Error while converting default value '{0}' to '{1}' member type. {2}".FormatString(defaultValue, ChoType.GetMemberName(memberInfo), ex.Message));
                }
            }

            _defaultsCache.Add(declaringType, dict);
        }
Beispiel #21
0
 public static void Error(string msg)
 {
     ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceError, msg);
 }
        private bool LoadLine(Tuple <int, string> pair, ref object rec)
        {
            Type recType = rec.GetType();

            try
            {
                if (!RaiseBeforeRecordLoad(rec, ref pair))
                {
                    return(false);
                }

                if (pair.Item2 == null)
                {
                    rec = null;
                    return(true);
                }
                else if (pair.Item2 == String.Empty)
                {
                    return(true);
                }

                if (!pair.Item2.IsNullOrWhiteSpace())
                {
                    var config = GetConfiguration(rec.GetType());
                    if (config == null)
                    {
                        throw new ChoParserException("No parser found to parse record line.");
                    }

                    if (config.GetType() == typeof(ChoCSVRecordConfiguration))
                    {
                        var r = ChoCSVReader.LoadText(rec.GetType(), pair.Item2, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize);
                        rec = r.FirstOrDefault <object>();
                    }
                    else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration))
                    {
                        var r = ChoFixedLengthReader.LoadText(rec.GetType(), pair.Item2, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize);
                        rec = r.FirstOrDefault <object>();
                    }
                    else
                    {
                        throw new ChoParserException("Unsupported record line found to parse.");
                    }
                }

                if (!RaiseAfterRecordLoad(rec, pair))
                {
                    return(false);
                }
            }
            catch (ChoParserException pEx)
            {
                throw new ChoParserException($"Failed to parse line to '{recType}' object.", pEx);
            }
            catch (ChoMissingRecordFieldException mEx)
            {
                throw new ChoParserException($"Failed to parse line to '{recType}' object.", mEx);
            }
            catch (Exception ex)
            {
                ChoETLFramework.HandleException(ex);
                if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                {
                    rec = null;
                }
                else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
                {
                    if (!RaiseRecordLoadError(rec, pair, ex))
                    {
                        throw new ChoParserException($"Failed to parse line to '{recType}' object.", ex);
                    }
                }
                else
                {
                    throw new ChoParserException($"Failed to parse line to '{recType}' object.", ex);
                }

                return(true);
            }

            return(true);
        }
Beispiel #23
0
 public static void Write(bool condition, string msg)
 {
     ChoETLFramework.WriteLog(condition, msg);
 }
        private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
        {
            TraceSwitch = traceSwitch;

            StreamReader sr = source as StreamReader;

            ChoGuard.ArgumentNotNull(sr, "StreamReader");

            if (Configuration.RecordSelector == null)
            {
                throw new ChoRecordConfigurationException("Missing record selector.");
            }

            sr.Seek(0, SeekOrigin.Begin);

            if (!RaiseBeginLoad(sr))
            {
                yield break;
            }

            string[] commentTokens = Configuration.Comments;
            bool     _headerFound  = false;

            using (ChoPeekEnumerator <Tuple <int, string> > e = new ChoPeekEnumerator <Tuple <int, string> >(
                       new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar)).ToEnumerable(),
                       (pair) =>
            {
                //bool isStateAvail = IsStateAvail();

                bool?skip = false;

                //if (isStateAvail)
                //{
                //    if (!IsStateMatches(item))
                //    {
                //        skip = filterFunc != null ? filterFunc(item) : false;
                //    }
                //    else
                //        skip = true;
                //}
                //else
                //    skip = filterFunc != null ? filterFunc(item) : false;

                if (skip == null)
                {
                    return(null);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //{
                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);

                if (!skip.Value)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
                }
                else
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
                }
                //}

                if (skip.Value)
                {
                    return(skip);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));

                //if (Task != null)
                //    return !IsStateNOTExistsOrNOTMatch(item);

                if (pair.Item2.IsNullOrWhiteSpace())
                {
                    if (!Configuration.IgnoreEmptyLine)
                    {
                        throw new ChoParserException("Empty line found at {0} location.".FormatString(e.Peek.Item1));
                    }
                    else
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Empty line found at [{0}]...".FormatString(pair.Item1));
                        return(true);
                    }
                }

                if (commentTokens == null)
                {
                    return(false);
                }
                else
                {
                    var x = (from comment in commentTokens
                             where !pair.Item2.IsNull() && pair.Item2.StartsWith(comment, true, Configuration.Culture)
                             select comment).FirstOrDefault();
                    if (x != null)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
                        return(true);
                    }
                }

                if (!_configCheckDone)
                {
                    Configuration.Validate(pair);     // GetHeaders(pair.Item2));
                    _configCheckDone = true;
                }

                //Ignore Header if any
                if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
                    !_headerFound)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
                    _headerFound = true;
                    return(true);
                }

                return(false);
            }))
            {
                while (true)
                {
                    Tuple <int, string> pair = e.Peek;
                    if (pair == null)
                    {
                        RaiseEndLoad(sr);
                        yield break;
                    }

                    Type recType = Configuration.RecordSelector(pair.Item2);
                    if (recType == null)
                    {
                        if (Configuration.IgnoreIfNoRecordParserExists)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
                        }
                        else
                        {
                            throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
                        }
                    }

                    object rec = ChoActivator.CreateInstance(recType);
                    if (!LoadLine(pair, ref rec))
                    {
                        yield break;
                    }

                    //StoreState(e.Current, rec != null);

                    e.MoveNext();

                    if (rec == null)
                    {
                        continue;
                    }

                    yield return(rec);
                }
            }
        }
Beispiel #25
0
        public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
        {
            TextWriter sw = writer as TextWriter;

            ChoGuard.ArgumentNotNull(sw, "TextWriter");

            if (records == null)
            {
                yield break;
            }

            if (!RaiseBeginWrite(sw))
            {
                yield break;
            }

            CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;

            System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;

            string recText  = String.Empty;
            long   recCount = 0;
            var    recEnum  = records.GetEnumerator();

            try
            {
                object notNullRecord = GetFirstNotNullRecord(recEnum);
                if (notNullRecord == null)
                {
                    yield break;
                }

                if (Configuration.IsDynamicObject)
                {
                    if (Configuration.MaxScanRows > 0)
                    {
                        List <string> fns = new List <string>();
                        foreach (object record1 in GetRecords(recEnum))
                        {
                            recCount++;

                            if (record1 != null)
                            {
                                if (recCount <= Configuration.MaxScanRows)
                                {
                                    if (!record1.GetType().IsDynamicType())
                                    {
                                        throw new ChoParserException("Invalid record found.");
                                    }

                                    _recBuffer.Value.Add(record1);
                                    fns = fns.Union(GetFields(record1)).ToList();

                                    if (recCount == Configuration.MaxScanRows)
                                    {
                                        Configuration.Validate(fns.ToArray());
                                        WriteHeaderLine(sw);
                                        _configCheckDone = true;
                                        break;
                                    }
                                }
                            }
                        }
                    }
                }

                foreach (object record in GetRecords(recEnum))
                {
                    _index++;

                    if (TraceSwitch.TraceVerbose)
                    {
                        if (record is IChoETLNameableObject)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
                        }
                        else
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
                        }
                    }

                    recText = String.Empty;
                    if (record != null)
                    {
                        if (predicate == null || predicate(record))
                        {
                            //Discover and load FixedLength columns from first record
                            if (!_configCheckDone)
                            {
                                if (notNullRecord != null)
                                {
                                    string[] fieldNames = GetFields(notNullRecord);
                                    Configuration.Validate(fieldNames);
                                    WriteHeaderLine(sw);
                                    _configCheckDone = true;
                                }
                            }
                            //Check record
                            if (record != null)
                            {
                                Type rt = record.GetType().ResolveType();
                                if (Configuration.IsDynamicObject)
                                {
                                    if (ElementType != null)
                                    {
                                    }
                                    else if (!rt.IsDynamicType())
                                    {
                                        throw new ChoWriterException("Invalid record found.");
                                    }
                                }
                                else
                                {
                                    if (rt != Configuration.RecordType)
                                    {
                                        throw new ChoWriterException("Invalid record found.");
                                    }
                                }
                            }

                            if (!RaiseBeforeRecordWrite(record, _index, ref recText))
                            {
                                yield break;
                            }

                            if (recText == null)
                            {
                                continue;
                            }
                            else if (recText.Length > 0)
                            {
                                sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : "");
                                continue;
                            }

                            try
                            {
                                if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
                                {
                                    record.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
                                }

                                if (ToText(_index, record, out recText))
                                {
                                    if (_index == 1)
                                    {
                                        sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : "");
                                    }
                                    else
                                    {
                                        sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
                                    }

                                    if (!RaiseAfterRecordWrite(record, _index, recText))
                                    {
                                        yield break;
                                    }
                                }
                            }
                            //catch (ChoParserException)
                            //{
                            //    throw;
                            //}
                            catch (Exception ex)
                            {
                                ChoETLFramework.HandleException(ref ex);
                                if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                                {
                                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                                }
                                else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
                                {
                                    if (!RaiseRecordWriteError(record, _index, recText, ex))
                                    {
                                        throw;
                                    }
                                    else
                                    {
                                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                                    }
                                }
                                else
                                {
                                    throw;
                                }
                            }
                        }
                    }

                    yield return(record);

                    if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
                    {
                        if (RaisedRowsWritten(_index))
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
                            yield break;
                        }
                    }
                }
            }
            finally
            {
                System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
            }

            RaiseEndWrite(sw);
        }
Beispiel #26
0
        private bool LoadLine(Tuple <long, string> pair, ref object rec)
        {
            try
            {
                if (!RaiseBeforeRecordLoad(rec, ref pair))
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
                    rec = null;
                    return(true);
                }

                if (pair.Item2 == null)
                {
                    rec = null;
                    return(true);
                }
                else if (pair.Item2 == String.Empty)
                {
                    return(true);
                }

                if (!pair.Item2.IsNullOrWhiteSpace())
                {
                    if (!FillRecord(rec, pair))
                    {
                        return(false);
                    }

                    if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
                    {
                        rec.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
                    }
                }

                bool skip = false;
                if (!RaiseAfterRecordLoad(rec, pair, ref skip))
                {
                    return(false);
                }
                else if (skip)
                {
                    rec = null;
                    return(true);
                }
            }
            //catch (ChoParserException)
            //{
            //    throw;
            //}
            //catch (ChoMissingRecordFieldException)
            //{
            //    throw;
            //}
            catch (Exception ex)
            {
                Reader.IsValid = false;
                if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
                {
                    throw;
                }

                ChoETLFramework.HandleException(ref ex);

                if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                    rec = null;
                }
                else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
                {
                    if (!RaiseRecordLoadError(rec, pair, ex))
                    {
                        throw;
                    }
                    else
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                        rec = null;
                    }
                }
                else
                {
                    throw;
                }

                return(true);
            }

            return(true);
        }
        private bool LoadLines(Tuple <long, List <Tuple <long, string> > > pairs, ref object rec)
        {
            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading [{0}] record...".FormatString(pairs.Item1));

            Tuple <long, string> pair = null;
            long recNo = pairs.Item1;

            if (!Configuration.AutoDiscoveredColumns)
            {
                if (Configuration.ColumnCountStrict)
                {
                    if (pairs.Item2.Count != Configuration.KVPRecordFieldConfigurations.Count)
                    {
                        throw new ChoParserException("Incorrect number of field values found at record [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.KVPRecordFieldConfigurations.Count, pairs.Item2.Count, recNo));
                    }
                    List <string> keys = (from p in pairs.Item2
                                          select ToKVP(p.Item1, p.Item2).Key).ToList();

                    if (!Enumerable.SequenceEqual(keys.OrderBy(t => t), Configuration.FCArray.Select(a => a.Value.FieldName).OrderBy(t => t), Configuration.FileHeaderConfiguration.StringComparer))
                    {
                        throw new ChoParserException("Column count mismatch detected at [{0}] record.".FormatString(recNo));
                    }
                }
                if (Configuration.ColumnOrderStrict)
                {
                    List <string> keys = (from p in pairs.Item2
                                          select ToKVP(p.Item1, p.Item2).Key).ToList();
                    int runnngIndex = -1;
                    foreach (var k in keys)
                    {
                        runnngIndex++;
                        if (runnngIndex < Configuration.FCArray.Length)
                        {
                            if (Configuration.FileHeaderConfiguration.IsEqual(Configuration.FCArray[runnngIndex].Value.FieldName, k))
                            {
                                continue;
                            }
                        }
                        throw new ChoParserException("Found incorrect order on '{1}' column at [{0}] record.".FormatString(recNo, k));
                    }
                }
            }

            object       fieldValue = String.Empty;
            PropertyInfo pi         = null;

            //Set default values
            foreach (KeyValuePair <string, ChoKVPRecordFieldConfiguration> kvp in Configuration.FCArray)
            {
                if (Configuration.PIDict != null)
                {
                    Configuration.PIDict.TryGetValue(kvp.Key, out pi);
                }
                try
                {
                    if (kvp.Value.IsDefaultValueSpecified)
                    {
                        fieldValue = kvp.Value.DefaultValue;
                    }

                    if (Configuration.IsDynamicObject)
                    {
                        var dict = rec as IDictionary <string, Object>;
                        dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
                    }
                    else
                    {
                        if (pi != null)
                        {
                            rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
                        }
                    }
                }
                catch
                {
                }
            }

            _propInit.Clear();
            foreach (var pair1 in pairs.Item2)
            {
                pair = pair1;
                try
                {
                    if (!RaiseBeforeRecordLoad(rec, ref pair))
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
                        rec = null;
                        return(true);
                    }

                    if (pair.Item2 == null)
                    {
                        rec = null;
                        return(true);
                    }
                    else if (pair.Item2 == String.Empty)
                    {
                        return(true);
                    }

                    if (!pair.Item2.IsNullOrWhiteSpace())
                    {
                        if (!FillRecord(rec, pair))
                        {
                            return(false);
                        }

                        if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
                        {
                            rec.DoObjectLevelValidation(Configuration, Configuration.KVPRecordFieldConfigurations);
                        }
                    }

                    bool skip = false;
                    if (!RaiseAfterRecordLoad(rec, pair, ref skip))
                    {
                        return(false);
                    }
                    else if (skip)
                    {
                        rec = null;
                        return(true);
                    }
                }
                //catch (ChoParserException)
                //{
                //    throw;
                //}
                //catch (ChoMissingRecordFieldException)
                //{
                //    throw;
                //}
                catch (Exception ex)
                {
                    if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
                    {
                        throw;
                    }

                    ChoETLFramework.HandleException(ref ex);
                    if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                        rec = null;
                    }
                    else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
                    {
                        if (!RaiseRecordLoadError(rec, pair, ex))
                        {
                            throw;
                        }
                        else
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
                            rec = null;
                        }
                    }
                    else
                    {
                        throw;
                    }

                    return(true);
                }
            }

            return(true);
        }
Beispiel #28
0
        private bool FillRecord(object rec, Tuple <long, string> pair)
        {
            long   lineNo;
            string line;

            lineNo = pair.Item1;
            line   = pair.Item2;

            if (line.Length != Configuration.RecordLength)
            {
                throw new ChoParserException("Incorrect record length [Length: {0}] found. Expected record length: {1}".FormatString(line.Length, Configuration.RecordLength));
            }

            object fieldValue = null;
            ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
            PropertyInfo pi      = null;
            object       rootRec = rec;

            foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
            {
                fieldValue  = null;
                fieldConfig = kvp.Value;
                if (Configuration.PIDict != null)
                {
                    Configuration.PIDict.TryGetValue(kvp.Key, out pi);
                }

                rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
                try
                {
                    if (fieldConfig.StartIndex + fieldConfig.Size > line.Length)
                    {
                        if (Configuration.ColumnCountStrict)
                        {
                            throw new ChoParserException("Missing '{0}' field value.".FormatString(kvp.Key));
                        }
                    }
                    else
                    {
                        fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value);
                    }

                    if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
                    {
                        if (kvp.Value.FieldType == null)
                        {
                            kvp.Value.FieldType = Configuration.MaxScanRows == -1 ? DiscoverFieldType(fieldValue as string, Configuration) : typeof(string);
                        }
                    }
                    else
                    {
                        if (pi != null)
                        {
                            kvp.Value.FieldType = pi.PropertyType;
                        }
                        else
                        {
                            kvp.Value.FieldType = typeof(string);
                        }
                    }

                    fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string);

                    if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
                    {
                        continue;
                    }

                    bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
                    if (ignoreFieldValue)
                    {
                        fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
                    }

                    if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
                    {
                        var dict = rec as IDictionary <string, Object>;

                        dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);

                        if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
                        {
                            dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
                        }
                    }
                    else
                    {
                        if (Configuration.SupportsMultiRecordTypes)
                        {
                            ChoType.TryGetProperty(rec.GetType(), kvp.Key, out pi);
                            fieldConfig.PI                  = pi;
                            fieldConfig.PropConverters      = ChoTypeDescriptor.GetTypeConverters(fieldConfig.PI);
                            fieldConfig.PropConverterParams = ChoTypeDescriptor.GetTypeConverterParams(fieldConfig.PI);
                        }

                        if (pi != null)
                        {
                            rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
                        }
                        else if (!Configuration.SupportsMultiRecordTypes)
                        {
                            throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
                        }

                        if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
                        {
                            rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
                        }
                    }

                    if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
                    {
                        return(false);
                    }
                }
                catch (ChoParserException)
                {
                    Reader.IsValid = false;
                    throw;
                }
                catch (ChoMissingRecordFieldException)
                {
                    Reader.IsValid = false;
                    if (Configuration.ThrowAndStopOnMissingField)
                    {
                        throw;
                    }
                }
                catch (Exception ex)
                {
                    Reader.IsValid = false;
                    ChoETLFramework.HandleException(ref ex);

                    if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
                    {
                        throw;
                    }

                    try
                    {
                        if (Configuration.IsDynamicObject)
                        {
                            var dict = rec as IDictionary <string, Object>;

                            if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
                            {
                                dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
                            }
                            else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
                            {
                                dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
                            }
                            else if (ex is ValidationException)
                            {
                                throw;
                            }
                            else
                            {
                                throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
                            }
                        }
                        else if (pi != null)
                        {
                            if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
                            {
                                rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
                            }
                            else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
                            {
                                rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
                            }
                            else if (ex is ValidationException)
                            {
                                throw;
                            }
                            else
                            {
                                throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
                            }
                        }
                        else
                        {
                            throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
                        }
                    }
                    catch (Exception innerEx)
                    {
                        if (ex == innerEx.InnerException || ex is ValidationException)
                        {
                            if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                            {
                                continue;
                            }
                            else
                            {
                                if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
                                {
                                    if (ex is ValidationException)
                                    {
                                        throw;
                                    }

                                    throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
                                }
                            }
                        }
                        else
                        {
                            throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
                        }
                    }
                }
            }

            return(true);
        }
        private bool FillRecord(object rec, Tuple <long, string> pair)
        {
            long   lineNo;
            string line;

            lineNo = pair.Item1;
            line   = pair.Item2;

            var tokens = ToKVP(pair.Item1, pair.Item2);

            //ValidateLine(pair.Item1, fieldValues);

            object fieldValue = tokens.Value;
            string key        = tokens.Key;

            if (!Configuration.RecordFieldConfigurationsDict2.ContainsKey(key))
            {
                return(true);
            }
            key = Configuration.RecordFieldConfigurationsDict2[key].Name;
            ChoKVPRecordFieldConfiguration fieldConfig = Configuration.RecordFieldConfigurationsDict[key];
            PropertyInfo pi = null;

            if (_propInit.ContainsKey(key))
            {
                return(true);
            }
            _propInit.Add(key, true);

            fieldValue = CleanFieldValue(fieldConfig, fieldConfig.FieldType, fieldValue as string);

            if (Configuration.IsDynamicObject)
            {
                if (fieldConfig.FieldType == null)
                {
                    fieldConfig.FieldType = typeof(string);
                }
            }
            else
            {
                if (Configuration.PIDict != null)
                {
                    Configuration.PIDict.TryGetValue(key, out pi);
                }

                if (pi != null)
                {
                    fieldConfig.FieldType = pi.PropertyType;
                }
                else
                {
                    fieldConfig.FieldType = typeof(string);
                }
            }

            fieldValue = CleanFieldValue(fieldConfig, fieldConfig.FieldType, fieldValue as string);

            if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, key, ref fieldValue))
            {
                return(true);
            }

            try
            {
                bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
                if (ignoreFieldValue)
                {
                    fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
                }

                if (Configuration.IsDynamicObject)
                {
                    var dict = rec as IDictionary <string, Object>;

                    dict.ConvertNSetMemberValue(key, fieldConfig, ref fieldValue, Configuration.Culture);

                    if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
                    {
                        dict.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
                    }
                }
                else
                {
                    if (pi != null)
                    {
                        rec.ConvertNSetMemberValue(key, fieldConfig, ref fieldValue, Configuration.Culture);
                    }
                    else
                    {
                        throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(key, ChoType.GetTypeName(rec)));
                    }

                    if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
                    {
                        rec.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
                    }
                }

                if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, key, fieldValue))
                {
                    return(false);
                }
            }
            catch (ChoParserException)
            {
                Reader.IsValid = false;
                throw;
            }
            catch (ChoMissingRecordFieldException)
            {
                Reader.IsValid = false;
                if (Configuration.ThrowAndStopOnMissingField)
                {
                    throw;
                }
            }
            catch (Exception ex)
            {
                Reader.IsValid = false;
                ChoETLFramework.HandleException(ref ex);

                if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
                {
                    throw;
                }

                try
                {
                    if (Configuration.IsDynamicObject)
                    {
                        var dict = rec as IDictionary <string, Object>;

                        if (dict.SetFallbackValue(key, fieldConfig, Configuration.Culture, ref fieldValue))
                        {
                            dict.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
                        }
                        else if (dict.SetDefaultValue(key, fieldConfig, Configuration.Culture))
                        {
                            dict.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
                        }
                        else
                        {
                            throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
                        }
                    }
                    else if (pi != null)
                    {
                        if (rec.SetFallbackValue(key, fieldConfig, Configuration.Culture))
                        {
                            rec.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
                        }
                        else if (rec.SetDefaultValue(key, fieldConfig, Configuration.Culture))
                        {
                            rec.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
                        }
                        else
                        {
                            throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
                        }
                    }
                    else
                    {
                        throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
                    }
                }
                catch (Exception innerEx)
                {
                    if (ex == innerEx.InnerException)
                    {
                        if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
                        {
                        }
                        else
                        {
                            if (!RaiseRecordFieldLoadError(rec, pair.Item1, key, fieldValue, ex))
                            {
                                throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
                            }
                        }
                    }
                    else
                    {
                        throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
                    }
                }
            }

            return(true);
        }
Beispiel #30
0
        private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
        {
            TraceSwitch = traceSwitch;

            TextReader sr = source as TextReader;

            ChoGuard.ArgumentNotNull(sr, "TextReader");

            if (sr is StreamReader)
            {
                ((StreamReader)sr).Seek(0, SeekOrigin.Begin);
            }

            if (!RaiseBeginLoad(sr))
            {
                yield break;
            }

            string[]      commentTokens              = Configuration.Comments;
            bool?         skip                       = false;
            bool          abortRequested             = false;
            long          runningCount               = 0;
            long          recCount                   = 0;
            bool          headerLineLoaded           = false;
            List <object> buffer                     = new List <object>();
            IDictionary <string, Type> recFieldTypes = null;
            bool?skipUntil = true;
            bool?doWhile   = true;

            using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
                       new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, false /*Configuration.MayContainEOLInData*/)).ToEnumerable(),
                       (pair) =>
            {
                //bool isStateAvail = IsStateAvail();
                skip = false;

                if (skipUntil != null)
                {
                    if (skipUntil.Value)
                    {
                        skipUntil = RaiseSkipUntil(pair);
                        if (skipUntil == null)
                        {
                        }
                        else
                        {
                            if (skipUntil.Value)
                            {
                                skip = skipUntil;
                            }
                            else
                            {
                                skip = true;
                            }
                        }
                    }
                }
                //if (isStateAvail)
                //{
                //    if (!IsStateMatches(item))
                //    {
                //        skip = filterFunc != null ? filterFunc(item) : false;
                //    }
                //    else
                //        skip = true;
                //}
                //else
                //    skip = filterFunc != null ? filterFunc(item) : false;

                if (skip == null)
                {
                    return(null);
                }


                if (TraceSwitch.TraceVerbose)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);

                    if (!skip.Value)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
                    }
                    else
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
                    }
                }

                if (skip.Value)
                {
                    return(skip);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));

                //if (Task != null)
                //    return !IsStateNOTExistsOrNOTMatch(item);

                if (pair.Item2.IsNullOrWhiteSpace())
                {
                    if (!Configuration.IgnoreEmptyLine)
                    {
                        throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
                    }
                    else
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
                        }
                        return(true);
                    }
                }

                if (commentTokens != null && commentTokens.Length > 0)
                {
                    foreach (string comment in commentTokens)
                    {
                        if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal))     //, true, Configuration.Culture))
                        {
                            if (TraceSwitch.TraceVerbose)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
                            }
                            return(true);
                        }
                    }
                }

                if (Configuration.FileHeaderConfiguration.HeaderLineAt > 0)
                {
                    if (pair.Item1 < Configuration.FileHeaderConfiguration.HeaderLineAt)
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Header line at {1}. Skipping [{0}] line...".FormatString(pair.Item1, Configuration.FileHeaderConfiguration.HeaderLineAt));
                        }
                        return(true);
                    }
                }

                if (!_configCheckDone)
                {
                    if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null && !Configuration.RecordTypeMapped)
                    {
                    }
                    else
                    {
                        Configuration.Validate(pair);                                 // GetHeaders(pair.Item2));
                    }
                    var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
                    RaiseMembersDiscovered(dict);
                    Configuration.UpdateFieldTypesIfAny(dict);
                    _configCheckDone = true;
                }

                //LoadHeader if any
                if ((Configuration.FileHeaderConfiguration.HasHeaderRecord ||
                     Configuration.FileHeaderConfiguration.HeaderLineAt > 0) &&
                    !_headerFound)
                {
                    if (Configuration.FileHeaderConfiguration.IgnoreHeader)
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
                        }
                    }
                    else
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
                        }

                        headerLineLoaded = true;
                        LoadHeaderLine(pair);
                    }
                    _headerFound = true;
                    return(true);
                }

                return(false);
            }))
            {
                while (true)
                {
                    recCount++;
                    Tuple <long, string> pair = e.Peek;
                    if (pair == null)
                    {
                        if (!abortRequested)
                        {
                            RaisedRowsLoaded(runningCount);
                        }

                        RaiseEndLoad(sr);
                        yield break;
                    }
                    runningCount = pair.Item1;

                    object rec = null;
                    if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null)
                    {
                        Type recType = Configuration.RecordSelector(pair);
                        if (recType == null)
                        {
                            if (Configuration.IgnoreIfNoRecordTypeFound)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
                                e.MoveNext();
                                continue;
                            }
                            else
                            {
                                throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
                            }
                        }

                        if (!Configuration.RecordTypeMapped)
                        {
                            Configuration.MapRecordFields(recType);
                            Configuration.Validate(null);
                        }

                        rec = recType.IsDynamicType() ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
                        {
                            ThrowExceptionIfPropNotExists = true,
                            AlternativeKeys = Configuration.AlternativeKeys
                        } : ChoActivator.CreateInstance(recType);
                    }
                    else
                    {
                        rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
                        {
                            ThrowExceptionIfPropNotExists = true,
                            AlternativeKeys = Configuration.AlternativeKeys
                        } : ChoActivator.CreateInstance(RecordType);
                    }

                    if (!LoadLine(pair, ref rec))
                    {
                        yield break;
                    }

                    //StoreState(e.Current, rec != null);

                    e.MoveNext();

                    if (rec == null)
                    {
                        continue;
                    }

                    if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
                    {
                        if (Configuration.AreAllFieldTypesNull && Configuration.AutoDiscoverFieldTypes && Configuration.MaxScanRows > 0 && recCount <= Configuration.MaxScanRows)
                        {
                            buffer.Add(rec);
                            RaiseRecordFieldTypeAssessment(recFieldTypes, (IDictionary <string, object>)rec, recCount == Configuration.MaxScanRows);
                            if (recCount == Configuration.MaxScanRows || e.Peek == null)
                            {
                                Configuration.UpdateFieldTypesIfAny(recFieldTypes);
                                var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
                                RaiseMembersDiscovered(dict);

                                foreach (object rec1 in buffer)
                                {
                                    yield return(ConvertToNestedObjectIfApplicable(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes)) as object, headerLineLoaded));
                                }
                            }
                        }
                        else
                        {
                            yield return(ConvertToNestedObjectIfApplicable(rec, headerLineLoaded));
                        }
                    }
                    else
                    {
                        yield return(rec);
                    }

                    if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
                    {
                        if (RaisedRowsLoaded(pair.Item1))
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
                            yield break;
                        }
                    }

                    if (doWhile != null)
                    {
                        doWhile = RaiseDoWhile(pair);
                        if (doWhile != null && doWhile.Value)
                        {
                            break;
                        }
                    }
                }
            }
        }