Esempio n. 1
0
        private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
        {
            TraceSwitch = traceSwitch;

            TextReader sr = source as TextReader;

            ChoGuard.ArgumentNotNull(sr, "TextReader");

            if (sr is StreamReader)
            {
                ((StreamReader)sr).Seek(0, SeekOrigin.Begin);
            }

            if (!RaiseBeginLoad(sr))
            {
                yield break;
            }

            string[]                     commentTokens      = Configuration.Comments;
            bool?                        skip               = false;
            bool                         isRecordStartFound = false;
            bool                         isRecordEndFound   = false;
            long                         seekOriginPos      = sr is StreamReader ? ((StreamReader)sr).BaseStream.Position : 0;
            List <string>                headers            = new List <string>();
            Tuple <long, string>         lastLine           = null;
            List <Tuple <long, string> > recLines           = new List <Tuple <long, string> >();
            long                         recNo              = 0;
            int  loopCount      = Configuration.AutoDiscoverColumns && Configuration.KVPRecordFieldConfigurations.Count == 0 ? 2 : 1;
            bool isHeaderFound  = loopCount == 1;
            bool IsHeaderLoaded = false;
            Tuple <long, string> pairIn;
            bool abortRequested = false;

            for (int i = 0; i < loopCount; i++)
            {
                if (i == 1)
                {
                    if (sr is StreamReader)
                    {
                        ((StreamReader)sr).Seek(seekOriginPos, SeekOrigin.Begin);
                    }
                    TraceSwitch = traceSwitch;
                }
                else
                {
                    TraceSwitch = ChoETLFramework.TraceSwitchOff;
                }
                lastLine = null;
                recLines.Clear();
                isRecordEndFound   = false;
                isRecordStartFound = false;

                using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
                           new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(),
                           (pair) =>
                {
                    //bool isStateAvail = IsStateAvail();
                    skip = false;

                    //if (isStateAvail)
                    //{
                    //    if (!IsStateMatches(item))
                    //    {
                    //        skip = filterFunc != null ? filterFunc(item) : false;
                    //    }
                    //    else
                    //        skip = true;
                    //}
                    //else
                    //    skip = filterFunc != null ? filterFunc(item) : false;

                    if (skip == null)
                    {
                        return(null);
                    }


                    if (TraceSwitch.TraceVerbose)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);

                        if (!skip.Value)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
                        }
                        else
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
                        }
                    }

                    if (skip.Value)
                    {
                        return(skip);
                    }

                    //if (!(sr.BaseStream is MemoryStream))
                    //    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));

                    //if (Task != null)
                    //    return !IsStateNOTExistsOrNOTMatch(item);

                    if (pair.Item2.IsNullOrWhiteSpace())
                    {
                        if (!Configuration.IgnoreEmptyLine)
                        {
                            throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
                        }
                        else
                        {
                            if (TraceSwitch.TraceVerbose)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
                            }
                            return(true);
                        }
                    }

                    if (commentTokens != null && commentTokens.Length > 0)
                    {
                        foreach (string comment in commentTokens)
                        {
                            if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal))     //, true, Configuration.Culture))
                            {
                                if (TraceSwitch.TraceVerbose)
                                {
                                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
                                }
                                return(true);
                            }
                        }
                    }

                    if (!_configCheckDone)
                    {
                        Configuration.Validate(null);
                        var dict = Configuration.KVPRecordFieldConfigurations.ToDictionary(i1 => i1.Name, i1 => i1.FieldType == null ? null : i1.FieldType);
                        RaiseMembersDiscovered(dict);
                        Configuration.UpdateFieldTypesIfAny(dict);
                        _configCheckDone = true;
                    }

                    return(false);
                }))
                {
                    while (true)
                    {
                        pairIn = e.Peek;

                        if (!isRecordStartFound)
                        {
                            if (pairIn == null)
                            {
                                break;
                            }

                            lastLine = null;
                            recLines.Clear();
                            isRecordEndFound   = false;
                            isRecordStartFound = true;
                            if (!Configuration.RecordStart.IsNullOrWhiteSpace())
                            {
                                //Move to record start
                                while (!(Configuration.IsRecordStartMatch(e.Peek.Item2)))
                                {
                                    e.MoveNext();
                                    if (e.Peek == null)
                                    {
                                        break;
                                    }
                                }
                            }
                            if (e.Peek != null)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record start found at [{0}] line...".FormatString(e.Peek.Item1));
                            }

                            e.MoveNext();
                            continue;
                        }
                        else
                        {
                            string recordEnd = !Configuration.RecordEnd.IsNullOrWhiteSpace() ? Configuration.RecordEnd : Configuration.RecordStart;
                            if (!recordEnd.IsNullOrWhiteSpace())
                            {
                                if (e.Peek == null)
                                {
                                    if (Configuration.RecordEnd.IsNullOrWhiteSpace())
                                    {
                                        isRecordEndFound = true;
                                    }
                                    else
                                    {
                                        break;
                                    }
                                }
                                else
                                {
                                    //Move to record start
                                    if (Configuration.IsRecordEndMatch(e.Peek.Item2))
                                    {
                                        isRecordEndFound   = true;
                                        isRecordStartFound = false;
                                        if (e.Peek != null)
                                        {
                                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record end found at [{0}] line...".FormatString(e.Peek.Item1));
                                        }
                                    }
                                }
                            }
                            else if (e.Peek == null)
                            {
                                isRecordEndFound = true;
                            }

                            if (!isHeaderFound)
                            {
                                //if (isRecordEndFound && headers.Count == 0)
                                //{
                                //    //throw new ChoParserException("Unexpected EOF found.");
                                //}
                                if (!isRecordEndFound)
                                {
                                    e.MoveNext();
                                    if (e.Peek != null)
                                    {
                                        //If line empty or line continuation, skip
                                        if (pairIn.Item2.IsNullOrWhiteSpace() || IsLineContinuationCharFound(pairIn.Item2)) //.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
                                        {
                                        }
                                        else
                                        {
                                            string header = ToKVP(pairIn.Item1, pairIn.Item2).Key;
                                            if (!header.IsNullOrWhiteSpace())
                                            {
                                                headers.Add(header);
                                            }
                                        }
                                    }
                                }
                                else
                                {
                                    Configuration.Validate(headers.ToArray());
                                    isHeaderFound    = true;
                                    isRecordEndFound = false;
                                    IsHeaderLoaded   = true;
                                    break;
                                }
                            }
                            else
                            {
                                if (!IsHeaderLoaded)
                                {
                                    Configuration.Validate(new string[] { });
                                    IsHeaderLoaded = true;
                                }

                                if (isRecordEndFound && recLines.Count == 0)
                                {
                                    //throw new ChoParserException("Unexpected EOF found.");
                                }
                                else if (!isRecordEndFound)
                                {
                                    e.MoveNext();
                                    if (e.Peek != null)
                                    {
                                        //If line empty or line continuation, skip
                                        if (pairIn.Item2.IsNullOrWhiteSpace())
                                        {
                                            if (!Configuration.IgnoreEmptyLine)
                                            {
                                                throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pairIn.Item1));
                                            }
                                            else
                                            {
                                                Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter);
                                                recLines.RemoveAt(recLines.Count - 1);
                                                recLines.Add(t);
                                            }
                                        }
                                        else if (IsLineContinuationCharFound(pairIn.Item2)) //pairIn.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
                                        {
                                            if (lastLine == null)
                                            {
                                                throw new ChoParserException("Unexpected line continuation found at {0} location.".FormatString(pairIn.Item1));
                                            }
                                            else
                                            {
                                                Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter + pairIn.Item2);
                                                recLines.RemoveAt(recLines.Count - 1);
                                                recLines.Add(t);
                                            }
                                        }
                                        else
                                        {
                                            lastLine = pairIn;
                                            recLines.Add(pairIn);
                                        }
                                    }
                                }
                                else
                                {
                                    object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
                                    {
                                        ThrowExceptionIfPropNotExists = true,
                                        AlternativeKeys = Configuration.AlternativeKeys
                                    } : Activator.CreateInstance(RecordType);
                                    if (!LoadLines(new Tuple <long, List <Tuple <long, string> > >(++recNo, recLines), ref rec))
                                    {
                                        yield break;
                                    }

                                    isRecordStartFound = false;
                                    //StoreState(e.Current, rec != null);

                                    if (!Configuration.RecordEnd.IsNullOrWhiteSpace())
                                    {
                                        e.MoveNext();
                                    }

                                    if (rec == null)
                                    {
                                        continue;
                                    }

                                    yield return(rec);

                                    if (Configuration.NotifyAfter > 0 && recNo % Configuration.NotifyAfter == 0)
                                    {
                                        if (RaisedRowsLoaded(recNo))
                                        {
                                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
                                            abortRequested = true;
                                            yield break;
                                        }
                                    }

                                    if (e.Peek == null)
                                    {
                                        break;
                                    }
                                }
                            }
                        }
                    }
                }
            }

            if (!abortRequested)
            {
                RaisedRowsLoaded(recNo, true);
            }
            RaiseEndLoad(sr);
        }
Esempio n. 2
0
        private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
        {
            TraceSwitch = traceSwitch;

            TextReader sr = source as TextReader;

            ChoGuard.ArgumentNotNull(sr, "TextReader");

            if (sr is StreamReader)
            {
                ((StreamReader)sr).Seek(0, SeekOrigin.Begin);
            }

            if (!RaiseBeginLoad(sr))
            {
                yield break;
            }

            string[]      commentTokens              = Configuration.Comments;
            bool?         skip                       = false;
            bool          abortRequested             = false;
            long          runningCount               = 0;
            long          recCount                   = 0;
            bool          headerLineLoaded           = false;
            List <object> buffer                     = new List <object>();
            IDictionary <string, Type> recFieldTypes = null;
            bool?skipUntil = true;
            bool?doWhile   = true;

            using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
                       new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, false /*Configuration.MayContainEOLInData*/)).ToEnumerable(),
                       (pair) =>
            {
                //bool isStateAvail = IsStateAvail();
                skip = false;

                if (skipUntil != null)
                {
                    if (skipUntil.Value)
                    {
                        skipUntil = RaiseSkipUntil(pair);
                        if (skipUntil == null)
                        {
                        }
                        else
                        {
                            if (skipUntil.Value)
                            {
                                skip = skipUntil;
                            }
                            else
                            {
                                skip = true;
                            }
                        }
                    }
                }
                //if (isStateAvail)
                //{
                //    if (!IsStateMatches(item))
                //    {
                //        skip = filterFunc != null ? filterFunc(item) : false;
                //    }
                //    else
                //        skip = true;
                //}
                //else
                //    skip = filterFunc != null ? filterFunc(item) : false;

                if (skip == null)
                {
                    return(null);
                }


                if (TraceSwitch.TraceVerbose)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);

                    if (!skip.Value)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
                    }
                    else
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
                    }
                }

                if (skip.Value)
                {
                    return(skip);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));

                //if (Task != null)
                //    return !IsStateNOTExistsOrNOTMatch(item);

                if (pair.Item2.IsNullOrWhiteSpace())
                {
                    if (!Configuration.IgnoreEmptyLine)
                    {
                        throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
                    }
                    else
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
                        }
                        return(true);
                    }
                }

                if (commentTokens != null && commentTokens.Length > 0)
                {
                    foreach (string comment in commentTokens)
                    {
                        if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal))     //, true, Configuration.Culture))
                        {
                            if (TraceSwitch.TraceVerbose)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
                            }
                            return(true);
                        }
                    }
                }

                if (Configuration.FileHeaderConfiguration.HeaderLineAt > 0)
                {
                    if (pair.Item1 < Configuration.FileHeaderConfiguration.HeaderLineAt)
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Header line at {1}. Skipping [{0}] line...".FormatString(pair.Item1, Configuration.FileHeaderConfiguration.HeaderLineAt));
                        }
                        return(true);
                    }
                }

                if (!_configCheckDone)
                {
                    if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null && !Configuration.RecordTypeMapped)
                    {
                    }
                    else
                    {
                        Configuration.Validate(pair);                                 // GetHeaders(pair.Item2));
                    }
                    var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
                    RaiseMembersDiscovered(dict);
                    Configuration.UpdateFieldTypesIfAny(dict);
                    _configCheckDone = true;
                }

                //LoadHeader if any
                if ((Configuration.FileHeaderConfiguration.HasHeaderRecord ||
                     Configuration.FileHeaderConfiguration.HeaderLineAt > 0) &&
                    !_headerFound)
                {
                    if (Configuration.FileHeaderConfiguration.IgnoreHeader)
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
                        }
                    }
                    else
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
                        }

                        headerLineLoaded = true;
                        LoadHeaderLine(pair);
                    }
                    _headerFound = true;
                    return(true);
                }

                return(false);
            }))
            {
                while (true)
                {
                    recCount++;
                    Tuple <long, string> pair = e.Peek;
                    if (pair == null)
                    {
                        if (!abortRequested)
                        {
                            RaisedRowsLoaded(runningCount);
                        }

                        RaiseEndLoad(sr);
                        yield break;
                    }
                    runningCount = pair.Item1;

                    object rec = null;
                    if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null)
                    {
                        Type recType = Configuration.RecordSelector(pair);
                        if (recType == null)
                        {
                            if (Configuration.IgnoreIfNoRecordTypeFound)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
                                e.MoveNext();
                                continue;
                            }
                            else
                            {
                                throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
                            }
                        }

                        if (!Configuration.RecordTypeMapped)
                        {
                            Configuration.MapRecordFields(recType);
                            Configuration.Validate(null);
                        }

                        rec = recType.IsDynamicType() ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
                        {
                            ThrowExceptionIfPropNotExists = true,
                            AlternativeKeys = Configuration.AlternativeKeys
                        } : ChoActivator.CreateInstance(recType);
                    }
                    else
                    {
                        rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
                        {
                            ThrowExceptionIfPropNotExists = true,
                            AlternativeKeys = Configuration.AlternativeKeys
                        } : ChoActivator.CreateInstance(RecordType);
                    }

                    if (!LoadLine(pair, ref rec))
                    {
                        yield break;
                    }

                    //StoreState(e.Current, rec != null);

                    e.MoveNext();

                    if (rec == null)
                    {
                        continue;
                    }

                    if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
                    {
                        if (Configuration.AreAllFieldTypesNull && Configuration.AutoDiscoverFieldTypes && Configuration.MaxScanRows > 0 && recCount <= Configuration.MaxScanRows)
                        {
                            buffer.Add(rec);
                            RaiseRecordFieldTypeAssessment(recFieldTypes, (IDictionary <string, object>)rec, recCount == Configuration.MaxScanRows);
                            if (recCount == Configuration.MaxScanRows || e.Peek == null)
                            {
                                Configuration.UpdateFieldTypesIfAny(recFieldTypes);
                                var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
                                RaiseMembersDiscovered(dict);

                                foreach (object rec1 in buffer)
                                {
                                    yield return(ConvertToNestedObjectIfApplicable(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes)) as object, headerLineLoaded));
                                }
                            }
                        }
                        else
                        {
                            yield return(ConvertToNestedObjectIfApplicable(rec, headerLineLoaded));
                        }
                    }
                    else
                    {
                        yield return(rec);
                    }

                    if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
                    {
                        if (RaisedRowsLoaded(pair.Item1))
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
                            yield break;
                        }
                    }

                    if (doWhile != null)
                    {
                        doWhile = RaiseDoWhile(pair);
                        if (doWhile != null && doWhile.Value)
                        {
                            break;
                        }
                    }
                }
            }
        }
Esempio n. 3
0
        private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
        {
            TraceSwitch = traceSwitch;

            TextReader sr = source as TextReader;

            ChoGuard.ArgumentNotNull(sr, "TextReader");

            if (sr is StreamReader)
            {
                ((StreamReader)sr).Seek(0, SeekOrigin.Begin);
            }

            if (Configuration.RecordSelector == null)
            {
                throw new ChoRecordConfigurationException("Missing record selector.");
            }

            if (!RaiseBeginLoad(sr))
            {
                yield break;
            }

            string[] commentTokens = Configuration.Comments;
            bool?    skip          = false;
            bool     _headerFound  = false;
            bool?    skipUntil     = true;
            bool?    doWhile       = true;

            using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
                       new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData, Configuration.MaxLineSize)).ToEnumerable(),
                       (pair) =>
            {
                //bool isStateAvail = IsStateAvail();

                skip = false;
                if (skipUntil != null)
                {
                    if (skipUntil.Value)
                    {
                        skipUntil = RaiseSkipUntil(pair);
                        if (skipUntil == null)
                        {
                        }
                        else
                        {
                            skip = skipUntil.Value;
                        }
                    }
                }

                //if (isStateAvail)
                //{
                //    if (!IsStateMatches(item))
                //    {
                //        skip = filterFunc != null ? filterFunc(item) : false;
                //    }
                //    else
                //        skip = true;
                //}
                //else
                //    skip = filterFunc != null ? filterFunc(item) : false;

                if (skip == null)
                {
                    return(null);
                }


                if (TraceSwitch.TraceVerbose)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);

                    if (!skip.Value)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
                    }
                    else
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
                    }
                }

                if (skip.Value)
                {
                    return(skip);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));

                //if (Task != null)
                //    return !IsStateNOTExistsOrNOTMatch(item);

                if (pair.Item2.IsNullOrWhiteSpace())
                {
                    if (!Configuration.IgnoreEmptyLine)
                    {
                        throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
                    }
                    else
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
                        }
                        return(true);
                    }
                }

                if (commentTokens != null && commentTokens.Length > 0)
                {
                    foreach (string comment in commentTokens)
                    {
                        if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal))     //, true, Configuration.Culture))
                        {
                            if (TraceSwitch.TraceVerbose)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
                            }
                            return(true);
                        }
                    }
                }

                if (!_configCheckDone)
                {
                    Configuration.Validate(pair);     // GetHeaders(pair.Item2));
                    _configCheckDone = true;
                }

                //Ignore Header if any
                if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
                    !_headerFound)
                {
                    if (TraceSwitch.TraceVerbose)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
                    }
                    _headerFound = true;
                    return(true);
                }

                return(false);
            }))
            {
                while (true)
                {
                    Tuple <long, string> pair = e.Peek;
                    if (pair == null)
                    {
                        RaiseEndLoad(sr);
                        yield break;
                    }

                    Type recType = Configuration.RecordSelector(pair.Item2);
                    if (recType == null)
                    {
                        if (Configuration.IgnoreIfNoRecordParserExists)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
                        }
                        else
                        {
                            throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
                        }
                    }

                    object rec = ChoActivator.CreateInstance(recType);
                    if (!LoadLine(pair, ref rec))
                    {
                        yield break;
                    }

                    //StoreState(e.Current, rec != null);

                    e.MoveNext();

                    if (rec == null)
                    {
                        continue;
                    }

                    yield return(rec);

                    if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
                    {
                        if (RaisedRowsLoaded(pair.Item1))
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
                            yield break;
                        }
                    }

                    if (doWhile != null)
                    {
                        doWhile = RaiseDoWhile(pair);
                        if (doWhile != null && doWhile.Value)
                        {
                            break;
                        }
                    }
                }
            }
        }
        private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
        {
            TraceSwitch = traceSwitch;

            StreamReader sr = source as StreamReader;

            ChoGuard.ArgumentNotNull(sr, "StreamReader");

            if (Configuration.RecordSelector == null)
            {
                throw new ChoRecordConfigurationException("Missing record selector.");
            }

            sr.Seek(0, SeekOrigin.Begin);

            if (!RaiseBeginLoad(sr))
            {
                yield break;
            }

            string[] commentTokens = Configuration.Comments;
            bool     _headerFound  = false;

            using (ChoPeekEnumerator <Tuple <int, string> > e = new ChoPeekEnumerator <Tuple <int, string> >(
                       new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar)).ToEnumerable(),
                       (pair) =>
            {
                //bool isStateAvail = IsStateAvail();

                bool?skip = false;

                //if (isStateAvail)
                //{
                //    if (!IsStateMatches(item))
                //    {
                //        skip = filterFunc != null ? filterFunc(item) : false;
                //    }
                //    else
                //        skip = true;
                //}
                //else
                //    skip = filterFunc != null ? filterFunc(item) : false;

                if (skip == null)
                {
                    return(null);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //{
                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);

                if (!skip.Value)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
                }
                else
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
                }
                //}

                if (skip.Value)
                {
                    return(skip);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));

                //if (Task != null)
                //    return !IsStateNOTExistsOrNOTMatch(item);

                if (pair.Item2.IsNullOrWhiteSpace())
                {
                    if (!Configuration.IgnoreEmptyLine)
                    {
                        throw new ChoParserException("Empty line found at {0} location.".FormatString(e.Peek.Item1));
                    }
                    else
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Empty line found at [{0}]...".FormatString(pair.Item1));
                        return(true);
                    }
                }

                if (commentTokens == null)
                {
                    return(false);
                }
                else
                {
                    var x = (from comment in commentTokens
                             where !pair.Item2.IsNull() && pair.Item2.StartsWith(comment, true, Configuration.Culture)
                             select comment).FirstOrDefault();
                    if (x != null)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
                        return(true);
                    }
                }

                if (!_configCheckDone)
                {
                    Configuration.Validate(pair);     // GetHeaders(pair.Item2));
                    _configCheckDone = true;
                }

                //Ignore Header if any
                if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
                    !_headerFound)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
                    _headerFound = true;
                    return(true);
                }

                return(false);
            }))
            {
                while (true)
                {
                    Tuple <int, string> pair = e.Peek;
                    if (pair == null)
                    {
                        RaiseEndLoad(sr);
                        yield break;
                    }

                    Type recType = Configuration.RecordSelector(pair.Item2);
                    if (recType == null)
                    {
                        if (Configuration.IgnoreIfNoRecordParserExists)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
                        }
                        else
                        {
                            throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
                        }
                    }

                    object rec = ChoActivator.CreateInstance(recType);
                    if (!LoadLine(pair, ref rec))
                    {
                        yield break;
                    }

                    //StoreState(e.Current, rec != null);

                    e.MoveNext();

                    if (rec == null)
                    {
                        continue;
                    }

                    yield return(rec);
                }
            }
        }
Esempio n. 5
0
        public static string ToTextAll <TRec>(IEnumerable <TRec> records, ChoXmlRecordConfiguration configuration = null, TraceSwitch traceSwitch = null, string xpath = null)
            where TRec : class
        {
            if (records == null)
            {
                return(null);
            }

            if (typeof(DataTable).IsAssignableFrom(typeof(TRec)))
            {
                StringBuilder xml = new StringBuilder();

                foreach (var dt in records.Take(1))
                {
                    using (var w = new ChoXmlWriter(xml, configuration))
                        w.Write(dt);
                }

                return(xml.ToString());
            }
            else if (typeof(IDataReader).IsAssignableFrom(typeof(TRec)))
            {
                StringBuilder xml = new StringBuilder();

                foreach (var dt in records.Take(1))
                {
                    using (var w = new ChoXmlWriter(xml, configuration))
                        w.Write(dt);
                }

                return(xml.ToString());
            }

            var pe = new ChoPeekEnumerator <TRec>(records, (Func <TRec, bool?>)null);

            if (configuration == null)
            {
                configuration = new ChoXmlRecordConfiguration();
            }

            configuration.IgnoreRootName = false;

            TRec record = pe.Peek;

            if (record != null)
            {
                if (configuration.NodeName.IsNullOrWhiteSpace())
                {
                    ChoDynamicObject rec1 = record as ChoDynamicObject;
                    if (rec1 != null)
                    {
                        configuration.NodeName = rec1.DynamicObjectName;
                        if (configuration.RootName.IsNullOrWhiteSpace())
                        {
                            configuration.RootName = configuration.NodeName.ToPlural();
                        }
                    }
                    else
                    {
                        XmlRootAttribute root     = ChoType.GetCustomAttribute <XmlRootAttribute>(record.GetType(), false);
                        string           nodeName = "XElement";
                        if (root != null && !root.ElementName.IsNullOrWhiteSpace())
                        {
                            nodeName = root.ElementName.Trim();
                        }
                        else
                        {
                            nodeName = record.GetType().Name;
                        }

                        if (configuration.RootName.IsNullOrWhiteSpace())
                        {
                            configuration.RootName = nodeName.ToPlural();
                        }
                        configuration.NodeName = nodeName;
                    }
                }
            }
            else
            {
                if (configuration.RootName.IsNullOrWhiteSpace())
                {
                    configuration.RootName = "Root";
                }
            }

            using (var stream = new MemoryStream())
                using (var reader = new StreamReader(stream))
                    using (var writer = new StreamWriter(stream))
                        using (var parser = new ChoXmlWriter <TRec>(writer, configuration)
                        {
                            TraceSwitch = traceSwitch == null ? ChoETLFramework.TraceSwitch : traceSwitch
                        })
                        {
                            //parser.Configuration.XPath = xpath;

                            parser.Write(pe.ToEnumerable());

                            parser.Close();

                            writer.Flush();
                            stream.Position = 0;

                            return(reader.ReadToEnd());
                        }
        }
Esempio n. 6
0
        private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
        {
            TraceSwitch = traceSwitch;

            TextReader sr = source as TextReader;

            ChoGuard.ArgumentNotNull(sr, "TextReader");

            if (sr is StreamReader)
            {
                ((StreamReader)sr).Seek(0, SeekOrigin.Begin);
            }

            if (!RaiseBeginLoad(sr))
            {
                yield break;
            }

            string[] commentTokens  = Configuration.Comments;
            bool?    skip           = false;
            bool     abortRequested = false;
            long     runningCount   = 0;

            using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
                       new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, false /*Configuration.MayContainEOLInData*/)).ToEnumerable(),
                       (pair) =>
            {
                //bool isStateAvail = IsStateAvail();
                skip = false;

                //if (isStateAvail)
                //{
                //    if (!IsStateMatches(item))
                //    {
                //        skip = filterFunc != null ? filterFunc(item) : false;
                //    }
                //    else
                //        skip = true;
                //}
                //else
                //    skip = filterFunc != null ? filterFunc(item) : false;

                if (skip == null)
                {
                    return(null);
                }


                if (TraceSwitch.TraceVerbose)
                {
                    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);

                    if (!skip.Value)
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
                    }
                    else
                    {
                        ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
                    }
                }

                if (skip.Value)
                {
                    return(skip);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //    ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));

                //if (Task != null)
                //    return !IsStateNOTExistsOrNOTMatch(item);

                if (pair.Item2.IsNullOrWhiteSpace())
                {
                    if (!Configuration.IgnoreEmptyLine)
                    {
                        throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
                    }
                    else
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
                        }
                        return(true);
                    }
                }

                if (commentTokens != null && commentTokens.Length > 0)
                {
                    foreach (string comment in commentTokens)
                    {
                        if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal))     //, true, Configuration.Culture))
                        {
                            if (TraceSwitch.TraceVerbose)
                            {
                                ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
                            }
                            return(true);
                        }
                    }
                }

                if (!_configCheckDone)
                {
                    Configuration.Validate(pair);     // GetHeaders(pair.Item2));
                    RaiseMembersDiscovered(Configuration.FixedLengthRecordFieldConfigurations.Select(i => new KeyValuePair <string, Type>(i.Name, i.FieldType == null ? typeof(string) : i.FieldType)).ToArray());
                    _configCheckDone = true;
                }

                //LoadHeader if any
                if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
                    !_headerFound)
                {
                    if (Configuration.FileHeaderConfiguration.IgnoreHeader)
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
                        }
                    }
                    else
                    {
                        if (TraceSwitch.TraceVerbose)
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
                        }
                        LoadHeaderLine(pair);
                    }
                    _headerFound = true;
                    return(true);
                }

                return(false);
            }))
            {
                while (true)
                {
                    Tuple <long, string> pair = e.Peek;
                    if (pair == null)
                    {
                        if (!abortRequested)
                        {
                            RaisedRowsLoaded(runningCount);
                        }

                        RaiseEndLoad(sr);
                        yield break;
                    }
                    runningCount = pair.Item1;

                    object rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
                    {
                        ThrowExceptionIfPropNotExists = true
                    } : Activator.CreateInstance(RecordType);
                    if (!LoadLine(pair, ref rec))
                    {
                        yield break;
                    }

                    //StoreState(e.Current, rec != null);

                    e.MoveNext();

                    if (rec == null)
                    {
                        continue;
                    }

                    yield return(rec);

                    if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
                    {
                        if (RaisedRowsLoaded(pair.Item1))
                        {
                            ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
                            yield break;
                        }
                    }
                }
            }
        }
Esempio n. 7
0
 public ChoStdDeferedObjectMemberDiscoverer(IEnumerable collection)
 {
     _collection = collection;
     _enumerator = new ChoPeekEnumerator <object>(_collection.OfType <object>(), (Func <object, bool?>)null);
     _enumerator.MembersDiscovered += (o, e) => MembersDiscovered.Raise(o, e);
 }
Esempio n. 8
0
        public override IEnumerable <object> AsEnumerable(object source, Func <object, bool?> filterFunc = null)
        {
            StreamReader sr = source as StreamReader;

            ChoGuard.ArgumentNotNull(sr, "StreamReader");

            sr.Seek(0, SeekOrigin.Begin);

            if (!RaiseBeginLoad(sr))
            {
                yield break;
            }

            string[] commentTokens = Configuration.Comments;

            using (ChoPeekEnumerator <Tuple <int, string> > e = new ChoPeekEnumerator <Tuple <int, string> >(
                       new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar)).ToEnumerable(),
                       (pair) =>
            {
                //bool isStateAvail = IsStateAvail();

                bool?skip = false;

                //if (isStateAvail)
                //{
                //    if (!IsStateMatches(item))
                //    {
                //        skip = filterFunc != null ? filterFunc(item) : false;
                //    }
                //    else
                //        skip = true;
                //}
                //else
                //    skip = filterFunc != null ? filterFunc(item) : false;

                if (skip == null)
                {
                    return(null);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //{
                ChoETLFramework.WriteLog(Environment.NewLine);

                if (!skip.Value)
                {
                    ChoETLFramework.WriteLog("Loading line [{0}]...".FormatString(pair.Item1));
                }
                else
                {
                    ChoETLFramework.WriteLog("Skipping line [{0}]...".FormatString(pair.Item1));
                }
                //}

                if (skip.Value)
                {
                    return(skip);
                }

                //if (!(sr.BaseStream is MemoryStream))
                //    ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));

                //if (Task != null)
                //    return !IsStateNOTExistsOrNOTMatch(item);

                if (pair.Item2.IsNullOrWhiteSpace())
                {
                    if (!Configuration.IgnoreEmptyLine)
                    {
                        throw new ChoParserException("Empty line found at {0} location.".FormatString(e.Peek.Item1));
                    }
                    else
                    {
                        ChoETLFramework.WriteLog("Empty line found at [{0}]...".FormatString(pair.Item1));
                        return(true);
                    }
                }

                //LoadExcelSeparator if any
                if (pair.Item1 == 1 &&
                    !_excelSeparatorFound)
                {
                    ChoETLFramework.WriteLog("Inspecting for excel separator at [{0}]...".FormatString(pair.Item1));
                    bool retVal = LoadExcelSeperatorIfAny(pair);
                    _excelSeparatorFound = true;

                    if (Configuration.HasExcelSeparator.HasValue &&
                        Configuration.HasExcelSeparator.Value &&
                        !retVal)
                    {
                        throw new ChoParserException("Missing excel separator header line in the file.");
                    }

                    if (retVal)
                    {
                        return(true);
                    }
                }

                if (commentTokens == null)
                {
                    return(false);
                }
                else
                {
                    var x = (from comment in commentTokens
                             where !pair.Item2.IsNull() && pair.Item2.StartsWith(comment, true, Configuration.Culture)
                             select comment).FirstOrDefault();
                    if (x != null)
                    {
                        ChoETLFramework.WriteLog("Comment line found at [{0}]...".FormatString(pair.Item1));
                        return(true);
                    }
                }

                //LoadHeader if any
                if (Configuration.CSVFileHeaderConfiguration.HasHeaderRecord &&
                    !_headerFound)
                {
                    ChoETLFramework.WriteLog("Loading header line at [{0}]...".FormatString(pair.Item1));
                    LoadHeaderLine(pair);
                    _headerFound = true;
                    return(true);
                }

                return(false);
            }))
            {
                while (true)
                {
                    Tuple <int, string> pair = e.Peek;
                    if (pair == null)
                    {
                        RaiseEndLoad(sr);
                        yield break;
                    }

                    object rec = ChoActivator.CreateInstance(RecordType);
                    if (!LoadLine(pair, ref rec))
                    {
                        yield break;
                    }

                    //StoreState(e.Current, rec != null);

                    e.MoveNext();

                    if (rec == null)
                    {
                        continue;
                    }

                    yield return(rec);
                }
            }
        }