Ejemplo n.º 1
0
 public ScanPattern()
 {
     mPatternValue            = null;
     mNextSameLeadChar        = null;
     mLeadChar                = '\0';
     this.DelimClassification = Text.Enums.DelimClassification.None;
 }
Ejemplo n.º 2
0
 public ScanPattern(ScanPattern Pattern)
     : this()
 {
     this.ReplacementValue      = Pattern.ReplacementValue;
     this.PatternValue          = Pattern.PatternValue;
     this.DelimClassification   = Pattern.DelimClassification;
     this.OverlapClassification = Pattern.OverlapClassification;
 }
Ejemplo n.º 3
0
        public void AddDistinct(ScanPattern Pattern, DelimClassification DelimClass)
        {
            int px = base.AddDistinct(Pattern.PatternValue);

            if (px != -1)
            {
                mWip.Add(DelimClass);
            }
        }
Ejemplo n.º 4
0
 public ScanWordCursor(
     TextWord Word, TextLocation WordBx, TextLocation DelimBx, ScanPattern DelimPattern)
     : this()
 {
     this.Word         = Word;
     this.WordBx       = WordBx;
     this.DelimBx      = DelimBx;
     this.DelimPattern = DelimPattern;
 }
Ejemplo n.º 5
0
 public ScanAtomCursor(
     AtomText AtomText, ScanPattern AtomPattern,
     RelativePosition Position = RelativePosition.At)
     : this()
 {
     this.AtomText    = AtomText;
     this.AtomPattern = AtomPattern;
     this.Position    = Position;
 }
 public ScanPatternResults(int InFoundPos, string InFoundPattern, int InAnyPatternIx)
 {
     mScanStartIx   = -1;
     mScannedString = null;
     mScanBoundsEx  = -1;
     mFoundPos      = InFoundPos;
     mFoundChar     = InFoundPattern[0];
     mFoundPattern  = InFoundPattern;
     mAnyPatternIx  = InAnyPatternIx;
     mFoundPat      = null;
 }
 public ScanPatternResults(int InFoundPos, char InFoundChar)
 {
     mScanStartIx   = -1;
     mScannedString = null;
     mScanBoundsEx  = -1;
     mFoundPos      = InFoundPos;
     mFoundChar     = InFoundChar;
     mFoundPattern  = null;
     mAnyPatternIx  = -1;
     mFoundPat      = null;
 }
        int mAnyPatternIx; // location of the found pattern in the array of any pattern
        // to scan for.

        public ScanPatternResults(int InNotFoundIx)
        {
            mScanStartIx   = -1;
            mScannedString = null;
            mScanBoundsEx  = -1;
            mFoundPos      = InNotFoundIx;
            mFoundChar     = null;
            mFoundPattern  = null;
            mAnyPatternIx  = -1;
            mFoundPat      = null;
        }
 public ScanPatternResults(int InFoundPos, ScanPattern InFoundPattern)
 {
     mScanStartIx   = -1;
     mScannedString = null;
     mScanBoundsEx  = -1;
     mFoundPos      = InFoundPos;
     mFoundChar     = InFoundPattern.LeadChar;
     mFoundPattern  = InFoundPattern.PatternValue;
     mAnyPatternIx  = InFoundPattern.ArrayPosition;
     mFoundPat      = InFoundPattern;
 }
Ejemplo n.º 10
0
        IsolateNumericLiteral(
            ScanStream ScanStream,
            TextTraits Traits,
            int Bx)
        {
            // for now, all numeric literals are simple integers.
            // have to expand to determine if a float, decimal, what the sign is,
            // what the precision is.
            LiteralType litType = LiteralType.Integer;

            string      litText  = null;
            ScanPattern foundPat = null;
            int         foundIx  = -1;

            // step from char to char. Look for a char that is not part of the
            // numeric literal.
            int ix    = Bx;
            int litEx = Bx;

            while (true)
            {
                if (ix >= ScanStream.Stream.Length)
                {
                    break;
                }
                char ch1 = ScanStream.Stream[ix];
                if (Char.IsDigit(ch1) == false)
                {
                    break;
                }

                litEx = ix;
                ix   += 1;
            }

            // isolate the numeric literal.
            int lx = litEx - Bx + 1;

            litText = ScanStream.Substring(Bx, lx);

            // isolate the delim that follows that numeric literal.
            int bx = litEx + 1;

            if (bx < ScanStream.Stream.Length)
            {
                var rv = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.DelimPatterns);
                foundPat = rv.Item1;
                foundIx  = rv.Item2;
            }

            return(new Tuple <LiteralType, string, ScanPattern, int>(
                       litType, litText, foundPat, foundIx));
        }
Ejemplo n.º 11
0
        public bool IsOpenBraced(ScanPattern InPattern)
        {
            DelimClassification dc = GetDelimClass(InPattern);

            if ((dc == DelimClassification.OpenContentBraced) ||
                (dc == DelimClassification.OpenNamedBraced))
            {
                return(true);
            }
            else
            {
                return(false);
            }
        }
        IsolateQuotedWord(
            ScanStream ScanStream,
            TextTraits Traits,
            int Bx)
        {
            LiteralType litType = LiteralType.none;
            string      litText = null;
            char        ch1     = ScanStream.Stream[Bx];

            ScanPattern foundPat = null;
            int         foundIx  = -1;
            int         quoteEx  = -1;

            // is start of a verbatim string literal
            if ((Traits.VerbatimLiteralPattern != null) &&
                (Traits.VerbatimLiteralPattern.Match(ScanStream.Stream, Bx)))
            {
                var rv = VerbatimLiteral.ScanCloseQuote(
                    ScanStream.Stream, Traits.VerbatimLiteralPattern, Bx);
                quoteEx = rv.Item1;
                litText = rv.Item2;
                litType = LiteralType.VerbatimString;
            }

            // is a quoted literal
            else if (Traits.IsQuoteChar(ch1) == true)
            {
                quoteEx = Scanner.ScanCloseQuote(ScanStream.Stream, Bx, Traits.QuoteEncapsulation);
                if (quoteEx != -1)
                {
                    int lx = quoteEx - Bx + 1;
                    litText = ScanStream.Substring(Bx, lx);

                    // correct the following at some point. Should be either string or
                    // char lit.
                    litType = LiteralType.String;
                }
            }

            // isolate the delim that follows that quoted word.
            {
                int bx = quoteEx + 1;
                var rv = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.DelimPatterns);
                foundPat = rv.Item1;
                foundIx  = rv.Item2;
            }

            return(new Tuple <LiteralType, string, ScanPattern, int>(
                       litType, litText, foundPat, foundIx));
        }
Ejemplo n.º 13
0
        public override bool Equals(object obj)
        {
            ScanPattern compareTo = obj as ScanPattern;

            if ((compareTo.DelimClassification == this.DelimClassification) &&
                (compareTo.PatternValue == this.PatternValue) &&
                (compareTo.UserCode == this.UserCode))
            {
                return(true);
            }
            else
            {
                return(false);
            }
        }
Ejemplo n.º 14
0
        public static bool IsPatternStartChar(this ScanPattern Pattern, char Char)
        {
            bool rv = false;

            if (Pattern == null)
            {
                rv = false;
            }
            else if (Char == Pattern.LeadChar)
            {
                rv = true;
            }
            else
            {
                rv = false;
            }
            return(rv);
        }
        public void AddFound(ScanPattern Pattern, int Pos, int Lgth)
        {
            this.Position = Pos;

            if (FoundPattern == null)
            {
                this.FoundPattern = new MatchScanPattern(Pattern, Pos, Lgth);
            }
            else
            {
                if (FoundPatterns == null)
                {
                    FoundPatterns = new MatchScanPatternList();
                    FoundPatterns.Add(this.FoundPattern);
                }

                {
                    MatchScanPattern matPat = new MatchScanPattern(Pattern, Pos, Lgth);
                    FoundPatterns.Add(matPat);
                }
            }
        }
        public void AddFound(ScanPattern Pattern, int Pos, int Lgth)
        {
            if (FoundPattern == null)
            {
                this.FoundPattern = Pattern;
                this.FoundPos     = Pos;
                this.MatchLgth    = Lgth;
            }
            else
            {
                if (FoundPatterns == null)
                {
                    FoundPatterns = new List <MatchScanPattern>();
                    MatchScanPattern matPat = new MatchScanPattern(FoundPattern, MatchLgth);
                    FoundPatterns.Add(matPat);
                }

                {
                    MatchScanPattern matPat = new MatchScanPattern(Pattern, Lgth);
                    FoundPatterns.Add(matPat);
                }
            }
        }
Ejemplo n.º 17
0
 public ScanPattern(string PatternValue, DelimClassification DelimClassification)
 {
     this.PatternValue        = PatternValue;
     mNextSameLeadChar        = null;
     this.DelimClassification = DelimClassification;
 }
Ejemplo n.º 18
0
        // ------------------------ ScanNextWord -------------------------
        // Scans to the next word in the string. ( a word being the text bounded by the
        // delimeter and whitespace characters as spcfd in the TextTraits argument )
        // Return null when end of string.
        public static ScanWordCursor ScanNextWord(
            ScanStream ScanStream,
            TextTraits Traits, ScanWordCursor CurrentWord)
        {
            // components of the next word.
            TextWord     wordPart   = null;
            TextLocation wordBx     = null;
            ScanPattern  nonWordPat = null;
            TextLocation nonWordLoc = null;
            int          nonWordIx  = -1;

            // stay at the current location. return copy of the cursor, but with stayatflag
            // turned off.
            if (CurrentWord.StayAtFlag == true)
            {
                nonWordPat = CurrentWord.DelimPattern;
                nonWordLoc = CurrentWord.DelimBx;
                wordPart   = CurrentWord.Word;
                wordBx     = CurrentWord.WordBx;
            }

            else
            {
                #region STEP1 setup the begin pos of the next word.
                // ----------------------------- STEP 1 ------------------------------
                // setup the begin pos of the next word.
                int bx;
                {
                    // calc scan start position
                    bx = ScanWord.CalcScanNextStart(ScanStream, Traits, CurrentWord);

                    // advance past whitespace
                    if (bx != -1)
                    {
                        bx = Scanner.ScanNotEqual(ScanStream.Stream, Traits.WhitespacePatterns, bx);
                    }
                }
                // end STEP 1.
                #endregion

                #region STEP 2. Isolate either numeric lib, quoted lit or scan to non word pattern
                // ------------------------------- STEP 2 ----------------------------------
                // Isolate either numeric literal, quoted literal or scan to the next non word
                // pattern.
                LiteralType?litType = null;
                string      litText = null;
                {
                    // got a decimal digit. isolate the numeric literal string.
                    if ((bx != -1) && (Char.IsDigit(ScanStream.Stream[bx]) == true))
                    {
                        var rv = ScanWord.IsolateNumericLiteral(ScanStream, Traits, bx);
                        litType    = rv.Item1;
                        litText    = rv.Item2;
                        nonWordPat = rv.Item3; // the non word pattern immed after numeric literal
                        nonWordIx  = rv.Item4; // pos of foundPat
                    }

                    // got something.  now scan forward for the pattern that delimits the word.
                    else if (bx != -1)
                    {
                        {
                            var rv = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.DelimPatterns);
                            nonWordPat = rv.Item1;
                            nonWordIx  = rv.Item2;
                        }

                        // got a quote char. Isolate the quoted string, then find the delim that follows
                        // the quoted string.
                        if ((nonWordPat != null) &&
                            (nonWordPat.DelimClassification == DelimClassification.Quote) &&
                            (nonWordIx == bx))
                        {
                            var rv = IsolateQuotedWord(ScanStream, Traits, nonWordIx);
                            litType    = rv.Item1;
                            litText    = rv.Item2;
                            nonWordPat = rv.Item3; // the non word pattern immed after quoted literal
                            nonWordIx  = rv.Item4; // pos of foundPat.
                        }
                    }
                }
                // end STEP 2.
                #endregion

                #region STEP 3 - setup wordBx and wordPart with the found word.
                {
                    // got nothing.
                    if (bx == -1)
                    {
                    }

                    // no delim found. word text all the way to the end.
                    else if (nonWordIx == -1)
                    {
                        var rv = ScanWord.IsolateWordText(
                            ScanStream, Traits, litType, litText, bx, null);
                        wordBx   = rv.Item1;
                        wordPart = rv.Item2;

#if skip
                        wordBx = new StreamLocation(bx).ToTextLocation(ScanStream);
                        if (litType != null)
                        {
                            wordPart = new TextWord(litText, WordClassification.Quoted, Traits);
                        }
                        else
                        {
                            wordPart = new TextWord(
                                ScanStream.Substring(bx), WordClassification.Identifier, Traits);
                        }
#endif
                    }

                    // got a word and a non word pattern.
                    else if (nonWordIx > bx)
                    {
                        var rv = ScanWord.IsolateWordText(
                            ScanStream, Traits, litType, litText, bx, nonWordIx);
                        wordBx   = rv.Item1;
                        wordPart = rv.Item2;

#if skip
                        wordBx = new StreamLocation(bx).ToTextLocation(ScanStream);
                        int lx = foundIx - bx;
                        wordPart = new TextWord(
                            ScanStream.Substring(bx, lx), WordClassification.Identifier, Traits);
#endif

                        nonWordLoc = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                    }

                    // no word. just delim.
                    else
                    {
                        nonWordLoc = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);

                        // the delim is comment to end. store as a word.
                        if (nonWordPat.DelimClassification == DelimClassification.CommentToEnd)
                        {
                            var rv     = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.NewLinePatterns);
                            var eolPat = rv.Item1;
                            var eolIx  = rv.Item2;
                            if (eolPat == null)
                            {
                                wordBx   = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                wordPart = new TextWord(
                                    ScanStream.Substring(nonWordIx), WordClassification.CommentToEnd, Traits);
                                nonWordLoc = null;
                                nonWordPat = null;
                            }
                            else
                            {
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                int lx   = eolIx - nonWordIx;
                                var sloc = wordBx.ToStreamLocation(ScanStream);
                                wordPart = new TextWord(
                                    ScanStream.Substring(sloc.Value, lx), WordClassification.CommentToEnd, Traits);
                                nonWordLoc = new StreamLocation(eolIx).ToTextLocation(ScanStream);
                                nonWordPat = eolPat;
                            }
                        }

                        // if the delim pattern is not non word ( a divider ), store the pattern also
                        // as the word.
                        else if (Traits.DelimPatternsThatAreNonWords.Contains(nonWordPat) == false)
                        {
                            wordBx   = nonWordLoc;
                            wordPart = new TextWord(
                                nonWordPat.PatternValue,
                                nonWordPat.DelimClassification.ToWordClassification().Value,
                                Traits);
                        }
                    }
                }
                #endregion

                // delim is whitespace. scan ahead for something more meaningful than whitespace.
                if ((nonWordPat != null) && (Traits.IsWhitespace(nonWordPat)))
                {
                    StreamLocation dx = nonWordLoc.ToStreamLocation(ScanStream);
                    int            fx = Scanner.ScanNotEqual(
                        ScanStream.Stream, Traits.WhitespacePatterns, dx.Value + nonWordPat.Length);
                    var pat = Traits.DelimPatterns.MatchAt(ScanStream.Stream, fx);
                    if (pat != null)
                    {
                        nonWordLoc = new StreamLocation(fx).ToTextLocation(ScanStream);
                        nonWordPat = pat;
                    }
                }
            }

            // store the results in the return cursor.
            ScanWordCursor nx = null;
            if ((wordPart == null) && (nonWordPat == null))
            {
                nx          = new ScanWordCursor( );
                nx.Position = RelativePosition.End;
            }
            else
            {
                nx          = new ScanWordCursor(wordPart, wordBx, nonWordLoc, nonWordPat);
                nx.Position = RelativePosition.At;
            }

            return(nx);
        }
Ejemplo n.º 19
0
 public MatchScanPattern(ScanPattern MatchPattern, int Pos, int MatchLength)
 {
     this.MatchPattern = MatchPattern;
     this.Pos          = Pos;
     this.MatchLength  = MatchLength;
 }
Ejemplo n.º 20
0
        /// <summary>
        /// Add the ScanPattern to the list of ScanPattern.
        /// That is, add to the list if another ScanPattern with the same delim class does
        /// not exist in the list.
        /// If a ScanPattern with the same delim class does exist, replace that ScanPattern
        /// with this one if the match length exceeds the pattern in the list.
        /// As an example, the ** pattern is found. It could be the start of a comment or the
        /// dereference of a pointer to a pointer. Also, a * could be a mult symbol, a
        /// dereference of a pointer, or the start of a special value.
        /// The ** that is a unary operator would replace the * unary operator in the list.
        /// The other entries in the list, the ones for delim class comment begin and
        /// special value starter would stay.
        /// </summary>
        /// <param name="Pat"></param>
        /// <param name="MatchLx"></param>
        public void Add(ScanPattern Pat, int Pos, int MatchLx)
        {
            MatchScanPattern matpat = new MatchScanPattern(Pat, Pos, MatchLx);

            Add(matpat);
        }
Ejemplo n.º 21
0
        // -------------------- IsolateDelim ---------------------------
        private static void IsolateDelim(
            string Text,
            ScanPatternResults PatternResults,
            ref WordCursor Results,
            TextTraits Traits)
        {
            // did not find a nonword char.  must have hit end of string.
            if (PatternResults.IsNotFound)
            {
                Results.DelimClass = DelimClassification.EndOfString;
            }

            // we have a delimiter of some kind.
            else
            {
                DelimClassification sprdc = PatternResults.FoundPat.DelimClassification;

                // delim is whitespace of some sort. Continue to look ahead for a non
                // whitespace pattern.
                if (Traits.IsWhitespaceDelimClass(sprdc) == true)
                {
                    int bx  = PatternResults.FoundPos;
                    var spr = Scanner.ScanNotEqual(
                        Text, bx, Text.Length - 1,
                        Traits.WhitespacePatterns);
                    if (spr.FoundPat != null)
                    {
                    }
                }

                Results.WhitespaceFollowsWord  = false;
                Results.WhitespaceFollowsDelim = false;
                Results.DelimIsWhitespace      = false;

                // the delim is a hard delim ( not whitespace )
                if (sprdc != DelimClassification.Whitespace)
                {
                    // Want the openContent brace to be processed as a standalone word. Use
                    // virtual whitespace so the word that this open brace is the delim of will
                    // have what appears to be a whitespace delim. Then the following word will
                    // be the standalone open content brace char.
                    if ((sprdc == DelimClassification.OpenContentBraced) &&
                        (Traits.VirtualWhitespace == true))
                    {
                        Results.SetDelim(
                            Text,
                            null, PatternResults.FoundPos, DelimClassification.VirtualWhitespace);
                    }
                    else
                    {
                        // delim is either as classified in the collection of NonWords or is
                        // a PathPart delim.
                        ScanPattern pat = Traits.GetPathPartDelim(
                            Text, PatternResults.FoundPos);
                        if (pat != null)
                        {
                            Results.SetDelim(
                                Text,
                                pat.PatternValue,
                                PatternResults.FoundPos,
                                DelimClassification.PathSep);
                        }
                        else
                        {
                            Results.SetDelim(
                                Text,
                                PatternResults.FoundPat.PatternValue,
                                PatternResults.FoundPos,
                                sprdc);
                        }
                    }
                }

                // whitespace immed follows the word text
                else
                {
                    ScanWord.IsolateDelim_WhitespaceFollows(
                        Text, PatternResults, ref Results, Traits);
                }
            }
        }
Ejemplo n.º 22
0
        /// <summary>
        /// The delim after the word is whitspace. If what follows the whitespace
        /// is a delim char, then this whitspace is disregarded as the delim, and
        /// the delim is what follows the whitespace.
        /// </summary>
        /// <param name="InBoundedString"></param>
        /// <param name="InNonWordResults"></param>
        /// <param name="InOutResults"></param>
        /// <param name="InTraits"></param>
        private static void IsolateDelim_WhitespaceFollows(
            string Text,
            ScanPatternResults PatternResults,
            ref WordCursor Results,
            TextTraits Traits)
        {
            Results.WhitespaceFollowsWord = true;
            ScanPattern nwPat     = null;
            int         nwMatchLx = 0;

            // Look for hard delim after the ws.
            ScanPatternResults scanResults =
                Scanner.ScanNotEqual(
                    Text, PatternResults.FoundPos, Text.Length - 1,
                    Traits.WhitespacePatterns);

            // look that the char after the ws is a nonword.
            if (scanResults.FoundPos != -1)
            {
                var rv = Traits.NonWordPatterns.MatchPatternsAtStringLocation(
                    Text, scanResults.FoundPos, Text.Length - 1);
                nwPat     = rv.Item1;
                nwMatchLx = rv.Item2;
            }

            // the char after the whitespace is a non word (delim) char.
            if (nwPat != null)
            {
                DelimClassification nwdc = nwPat.DelimClassification;

                // is the delim actually a sep char in a path name.
                // so the delim is the whitespace.
                if (Traits.IsPathPartDelim(Text, scanResults.FoundPos))
                {
                    ScanWord.IsolateDelim_SetDelimIsWhitespace(
                        Text, Traits, Results, PatternResults.FoundPos);
                }

                // is a content open brace char. delim stays as whitespace because
                // content braces are considered standalone words.
                else if (nwPat.DelimClassification.IsOpenBraced( ))
                {
                    ScanWord.IsolateDelim_SetDelimIsWhitespace(
                        Text, Traits, Results, PatternResults.FoundPos);
                }

                // is a quote char. the quoted string is considered a word.
                else if (nwdc == DelimClassification.Quote)
                {
                    ScanWord.IsolateDelim_SetDelimIsWhitespace(
                        Text, Traits, Results, PatternResults.FoundPos);
                }

                // is an actual delim.
                else
                {
                    Results.SetDelim(
                        Text,
                        nwPat.PatternValue, scanResults.FoundPos, nwdc);
                }
            }

            // the whitespace char is the delim of record.
            else
            {
                ScanWord.IsolateDelim_SetDelimIsWhitespace(
                    Text, Traits, Results, PatternResults.FoundPos);
            }
        }
 public FoundNonWord(ScanPattern Pattern, int Pos, int Lgth)
 {
     this.AddFound(Pattern, Pos, Lgth);
 }
Ejemplo n.º 24
0
        public virtual ScanPattern Duplicate()
        {
            ScanPattern dup = new ScanPattern(this);

            return(dup);
        }
Ejemplo n.º 25
0
        // ------------------------ ScanNextAtom -------------------------
        // Scans to the next atom in the string. ( a word being the text bounded by the
        // delimeter and whitespace characters as spcfd in the TextTraits argument )
        // Return null when end of string.
        public static ScanAtomCursor ScanNextAtom(
            ScanStream ScanStream,
            TextTraits Traits, ScanAtomCursor CurrentWord)
        {
            // components of the next word.
            TextLocation wordBx    = null;
            int          nonWordIx = -1;
            int          nonWordLx = 0;

            ScanPattern             nonWordPat     = null;
            List <MatchScanPattern> nonWordPatList = null;
            AtomText atomText = null;
            List <MatchScanPattern> atomTextList = null;
            AtomText whitespaceText = null;
//      ScanAtomCode? priorCode = null;
            bool?priorCodeIsWhitespaceSignificant = null;

            // stay at the current location. return copy of the cursor, but with stayatflag
            // turned off.
            if (CurrentWord.StayAtFlag == true)
            {
                atomText   = CurrentWord.AtomText;
                nonWordPat = CurrentWord.AtomPattern;
                wordBx     = CurrentWord.StartLoc;
            }

            else
            {
                #region STEP1 setup the begin pos of the next word.
                // ----------------------------- STEP 1 ------------------------------
                // setup the begin pos of the next word.
                int bx;
                {
                    // save the ScanAtomCode of the prior word.
                    if ((CurrentWord.Position == RelativePosition.At) ||
                        (CurrentWord.Position == RelativePosition.After))
                    {
                        priorCodeIsWhitespaceSignificant = CurrentWord.WhitespaceIsSignificant;
//            priorCode = CurrentWord.AtomText.AtomCode;
                    }

                    // calc scan start position
                    bx = ScanAtom.CalcScanNextStart(ScanStream, Traits, CurrentWord);

                    // advance past whitespace
                    if (bx != -1)
                    {
                        int saveBx = bx;
                        bx = Scanner.ScanNotEqual(ScanStream.Stream, Traits.WhitespacePatterns, bx);

                        // there is some whitespace. depending on what preceeds and follows, may
                        // return this as the atom.
                        if ((priorCodeIsWhitespaceSignificant != null) &&
                            (priorCodeIsWhitespaceSignificant.Value == true))
                        {
                            if (bx != saveBx)
                            {
                                int whitespaceEx = -1;
                                if (bx == -1)
                                {
                                    whitespaceEx = ScanStream.Stream.Length - 1;
                                }
                                else
                                {
                                    whitespaceEx = bx - 1;
                                }
                                int whitespaceLx = whitespaceEx - saveBx + 1;

                                whitespaceText = new AtomText(
                                    ScanAtomCode.Whitespace,
                                    ScanStream.Stream.Substring(saveBx, whitespaceLx), " ",
                                    new StreamLocation(saveBx).ToTextLocation(ScanStream),
                                    new StreamLocation(whitespaceEx).ToTextLocation(ScanStream));
                            }
                        }
                    }
                }
                // end STEP 1.
                #endregion

                #region STEP 2. Isolate either numeric lib, quoted lit or scan to non word pattern
                // ------------------------------- STEP 2 ----------------------------------
                // Isolate either numeric literal, quoted literal or scan to the next non word
                // pattern.
                LiteralType?litType = null;
                string      litText = null;
                {
                    // got a decimal digit. isolate the numeric literal string.
                    if ((bx != -1) && (Char.IsDigit(ScanStream.Stream[bx]) == true))
                    {
                        var rv = Scanner.IsolateNumericLiteral(ScanStream, Traits, bx);
                        litType    = rv.Item1;
                        litText    = rv.Item2;
                        nonWordPat = rv.Item3; // the non word pattern immed after numeric literal
                        nonWordIx  = rv.Item4; // pos of foundPat
                    }

                    // got something.  now scan forward for the pattern that delimits the word.
                    else if (bx != -1)
                    {
                        {
                            var rv = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.DelimPatterns);
                            nonWordPat     = rv.Item1;
                            nonWordIx      = rv.Item2;
                            nonWordLx      = rv.Item3;
                            nonWordPatList = rv.Item4;
                        }

                        // got a quote char. Isolate the quoted string, then find the delim that follows
                        // the quoted string.
                        if ((nonWordPat != null) &&
                            (nonWordPat.DelimClassification == DelimClassification.Quote) &&
                            (nonWordIx == bx))
                        {
                            var rv = Scanner.IsolateQuotedWord(ScanStream, Traits, nonWordIx);
                            litType    = rv.Item1;
                            litText    = rv.Item2;
                            nonWordPat = rv.Item3; // the non word pattern immed after quoted literal
                            nonWordIx  = rv.Item4; // pos of foundPat.
                        }
                    }
                }
                // end STEP 2.
                #endregion

                #region STEP 3 - setup wordBx and wordPart with the found word.
                {
                    // got nothing.
                    if (bx == -1)
                    {
                    }

                    // no delim found. word text all the way to the end.
                    else if (nonWordIx == -1)
                    {
                        if (whitespaceText != null)
                        {
                            atomText       = whitespaceText;
                            nonWordPat     = null;
                            nonWordPatList = null;
                        }

                        else
                        {
                            var rv = Scanner.IsolateWordText(
                                ScanStream, Traits, litType, litText, bx, null);
                            atomText = rv.Item3;
                            wordBx   = atomText.StartLoc;
                        }
                    }

                    // got a word and a non word pattern.
                    else if (nonWordIx > bx)
                    {
                        if (whitespaceText != null)
                        {
                            atomText       = whitespaceText;
                            nonWordPat     = null;
                            nonWordPatList = null;
                        }

                        else
                        {
                            var rv = Scanner.IsolateWordText(
                                ScanStream, Traits, litType, litText, bx, nonWordIx);
                            atomText = rv.Item3;
                            wordBx   = atomText.StartLoc;
                        }
                    }

                    // no word. just delim.
                    else
                    {
                        // the delim is comment to end. store as a word.
                        if (nonWordPat.DelimClassification == DelimClassification.CommentToEnd)
                        {
                            var rv     = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.NewLinePatterns);
                            var eolPat = rv.Item1;
                            var eolIx  = rv.Item2;
                            if (eolPat == null)
                            {
                                int ex = ScanStream.Stream.Length - 1;
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                TextLocation wordEx      = new StreamLocation(ex).ToTextLocation(ScanStream);
                                string       commentText = ScanStream.Substring(nonWordIx);

                                atomText = new AtomText(
                                    ScanAtomCode.CommentToEnd, commentText, null, wordBx, wordEx);

                                nonWordPat     = null;
                                nonWordPatList = null;
                            }
                            else
                            {
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                int          lx     = eolIx - nonWordIx;
                                TextLocation wordEx =
                                    new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                string commentText = ScanStream.Substring(nonWordIx, lx);
                                atomText = new AtomText(
                                    ScanAtomCode.CommentToEnd, commentText, null, wordBx, wordEx);
                                var sloc = wordBx.ToStreamLocation(ScanStream);

                                nonWordPat     = eolPat;
                                nonWordPatList = null;
                            }
                        }

                        // the word found is a non word or keyword pattern.
                        else
                        {
                            // got whitespace followed by keyword. Return the whitespace.
                            if ((nonWordPat.DelimClassification == DelimClassification.Keyword) &&
                                (whitespaceText != null))
                            {
                                atomText       = whitespaceText;
                                nonWordPat     = null;
                                nonWordPatList = null;
                            }

                            // there are more than one scan patterns that match.
                            else if (nonWordPatList != null)
                            {
                                atomTextList = new List <MatchScanPattern>();
                                foreach (var pat in nonWordPatList)
                                {
                                    wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                    int          lx     = pat.MatchLength;
                                    TextLocation wordEx =
                                        new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                    string scanText = ScanStream.Stream.Substring(nonWordIx, lx);

                                    atomText = new AtomText(
                                        pat.MatchPattern.DelimClassification.ToScanAtomCode().Value,
                                        scanText,
                                        pat.MatchPattern.ReplacementValue,
                                        wordBx, wordEx);

                                    pat.AtomText = atomText;
                                    atomTextList.Add(pat);
                                }
                            }

                            else
                            {
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                int          lx     = nonWordLx;
                                TextLocation wordEx =
                                    new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                string scanText = ScanStream.Stream.Substring(nonWordIx, lx);

                                atomText = new AtomText(
                                    nonWordPat.DelimClassification.ToScanAtomCode().Value,
                                    scanText, nonWordPat.ReplacementValue,
                                    wordBx, wordEx);
                            }
                        }
                    }
                }
                #endregion
            }

            // store the results in the return cursor.
            ScanAtomCursor nx = null;
            if (atomText == null)
            {
                nx          = new ScanAtomCursor( );
                nx.Position = RelativePosition.End;
            }
            else if (atomTextList != null)
            {
                nx = new ScanAtomCursor(atomTextList);
            }
            else
            {
                nx          = new ScanAtomCursor(atomText, nonWordPat);
                nx.Position = RelativePosition.At;
            }

            return(nx);
        }
 public PatternScanResults(ScanPattern Pattern, int Pos, int Lgth)
 {
     this.AddFound(Pattern, Pos, Lgth);
 }
Ejemplo n.º 27
0
        public DelimClassification GetDelimClass(ScanPattern InPattern)
        {
            DelimClassification dc = Delims[InPattern.ArrayPosition];

            return(dc);
        }
Ejemplo n.º 28
0
        ClassifyAsComment(ScanStream ScanStream, TextTraits Traits, int Bx)
        {
            TextLocation       wordBx     = null;
            AtomText           atomText   = null;
            ScanPattern        nonWordPat = null;
            PatternScanResults nonWord    = null;

            // look prior to see if this comment to the end of the line is the first non
            // blank on the line.
            bool isFirstNonBlankOnLine = false;

            if (Bx == 0)
            {
                isFirstNonBlankOnLine = true;
            }
            else
            {
                // go back to the first non blank.
                int ix = Scanner.ScanReverseNotEqual(
                    ScanStream.Stream, Bx - 1, Traits.WhitespaceWithoutNewLinePatterns);
                if (ix == -1) // nothing but blanks to start of string.
                {
                    isFirstNonBlankOnLine = true;
                }

                else
                {
                    var rv = Traits.NewLinePatterns.MatchFirstPatternEndsAtStringLocation(
                        ScanStream.Stream, ix);
                    var pat   = rv.Item1;
                    var patBx = rv.Item2;

                    // is a new line pattern. there is nothing but spaces between this new line
                    // and the start of the comment.
                    if (pat != null)
                    {
                        isFirstNonBlankOnLine = true;
                    }
                }
            }

            // set the atomCode of this atom depending on if the comment starts the line.
            ScanAtomCode atomCode = ScanAtomCode.CommentToEnd;

            if (isFirstNonBlankOnLine == true)
            {
                atomCode = ScanAtomCode.EntireLineCommentToEnd;
            }

            // scan for a new line. That is the end of the comment.
            {
                nonWord = Scanner.ScanEqualAny(ScanStream.Stream, Bx, Traits.NewLinePatterns);
//        eolPat = rv.Item1;
//        eolIx = rv.Item2;
//        nonWord = rv.Item3;
            }

            // no newline pattern found. Comment to the end of the text stream.
            if (nonWord.IsEmpty == true)
//      if (eolPat == null)
            {
                int ex = ScanStream.Stream.Length - 1;
                wordBx = new StreamLocation(Bx).ToTextLocation(ScanStream);
                TextLocation wordEx      = new StreamLocation(ex).ToTextLocation(ScanStream);
                string       commentText = ScanStream.Substring(Bx);

                string userCode = null;
                atomText = new AtomText(
                    atomCode, commentText, null, wordBx, wordEx,
                    userCode);

//        nonWordPat = eolPat;
            }

            else
            {
                wordBx = new StreamLocation(Bx).ToTextLocation(ScanStream);
                int lx = nonWord.Position - Bx;
//        int lx = eolIx - Bx;
                TextLocation wordEx =
                    new StreamLocation(Bx + lx - 1).ToTextLocation(ScanStream);
                string commentText = ScanStream.Substring(Bx, lx);
                string userCode    = null;
                atomText = new AtomText(
                    atomCode, commentText, null, wordBx, wordEx,
                    userCode);
                var sloc = wordBx.ToStreamLocation(ScanStream);

//        nonWordPat = eolPat;
            }
            return(new Tuple <TextLocation, AtomText, ScanPattern, PatternScanResults>
                       (wordBx, atomText, nonWordPat, nonWord));
        }
Ejemplo n.º 29
0
        // ------------------------ ScanNextAtom -------------------------
        // Scans to the next atom in the string. ( a word being the text bounded by the
        // delimiter and whitespace characters as spcfd in the TextTraits argument )
        // Return null when end of string.
        public static ScanAtomCursor OrigScanNextAtom(
            ScanStream ScanStream,
            TextTraits Traits, ScanAtomCursor CurrentWord)
        {
            // components of the next word.
            TextLocation wordBx    = null;
            int          nonWordIx = -1;
            int          nonWordLx = 0;

            ScanPattern        nonWordPat = null;
            PatternScanResults nonWord    = null;

            AtomText atomText = null;
            List <MatchScanPattern> atomTextList = null;
            AtomText whitespaceText = null;

            ScanAtomCode?tokenCode      = null; // ScanAtomCode of this token.
            int?         tokenLx        = null;
            ScanAtomCode?priorTokenCode = null;

            //      ScanAtomCode? priorCode = null;
            bool?priorCodeIsWhitespaceSignificant = null;

            // stay at the current location. return copy of the cursor, but with stayatflag
            // turned off.
            if (CurrentWord.StayAtFlag == true)
            {
                atomText       = CurrentWord.AtomText;
                tokenCode      = atomText.AtomCode;
                priorTokenCode = null;
                nonWordPat     = CurrentWord.AtomPattern;
                wordBx         = CurrentWord.StartLoc;
            }

            else
            {
                #region STEP1 setup the begin pos of the next word.
                // ----------------------------- STEP 1 ------------------------------
                // setup the begin pos of the next word.
                int bx;
                {
                    // save the ScanAtomCode of the prior word.
                    if ((CurrentWord.Position == RelativePosition.At) ||
                        (CurrentWord.Position == RelativePosition.After))
                    {
                        priorTokenCode = CurrentWord.AtomCode;
                        priorCodeIsWhitespaceSignificant = CurrentWord.WhitespaceIsSignificant;
                    }

                    // calc scan start position
                    bx = ScanAtom.CalcScanNextStart(ScanStream, Traits, CurrentWord);

                    // advance past whitespace
                    if (bx != -1)
                    {
                        int saveBx = bx;
                        bx = Scanner.ScanNotEqual(ScanStream.Stream, Traits.WhitespacePatterns, bx);

                        // there is some whitespace. Isolate it as AtomText.
                        // This method will return the whitespace as the token. But need to look at
                        // the token before and after to classify the whitespace as significant or
                        // not. ( whitespace between identifiers or keywords is significant.
                        // Whitespace between symbols is not significant.
                        // note: even insignificant whitespace is returned as a token because the
                        //       whitespace is needed when redisplaying the statement text.
                        if (bx != saveBx)
                        {
                            int whitespaceEx = -1;
                            if (bx == -1)
                            {
                                whitespaceEx = ScanStream.Stream.Length - 1;
                            }
                            else
                            {
                                whitespaceEx = bx - 1;
                            }
                            int whitespaceLx = whitespaceEx - saveBx + 1;

                            string userCode = null;
                            whitespaceText = new AtomText(
                                ScanAtomCode.Whitespace,
                                ScanStream.Stream.Substring(saveBx, whitespaceLx), " ",
                                new StreamLocation(saveBx).ToTextLocation(ScanStream),
                                new StreamLocation(whitespaceEx).ToTextLocation(ScanStream),
                                userCode);
                        }
                    }
                }
                // end STEP 1.
                #endregion

                #region STEP 2. Isolate either numeric lit, quoted lit or identifier/keyword.
                // ------------------------------- STEP 2 ----------------------------------
                // Isolate either numeric literal, quoted literal or scan to the next non word
                // pattern.
                LiteralType?litType = null;
                string      litText = null;
                {
                    // got a decimal digit. isolate the numeric literal string.
                    if ((bx != -1) && (Char.IsDigit(ScanStream.Stream[bx]) == true))
                    {
                        var rv = Scanner.IsolateNumericLiteral(ScanStream, Traits, bx);
                        litType = rv.Item1;
                        litText = rv.Item2;
                        nonWord = rv.Item3;
//            nonWordPat = rv.Item4;  // the non word pattern immed after numeric literal
//            nonWordIx = rv.Item5;   // pos of foundPat
                        tokenCode = ScanAtomCode.Numeric;
                    }

                    // got something.  now scan forward for the pattern that delimits the word.
                    else if (bx != -1)
                    {
                        {
                            nonWord = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.DelimPatterns);
//              nonWordPat = rv.Item1;
//              nonWordIx = rv.Item2;
//              nonWordLx = rv.Item3;
//              nonWord = rv.Item3;
                        }

                        // a special value starter. scan further for the spcval word.
                        var startPat = nonWord.FindPattern(DelimClassification.SpecialValueStarter);
                        if (startPat != null)
                        {
                        }

                        // word chars all the way to the end.
                        //            if (nonWordPat == null)
                        if (nonWord == null)
                        {
                            tokenCode = ScanAtomCode.Identifier;
                            tokenLx   = ScanStream.Stream.Length - bx;
                        }

                        else if (nonWord.FoundAtPosition(DelimClassification.Quote, bx))
                        {
                            var rv = Scanner.IsolateQuotedWord(ScanStream, Traits, bx);
                            litType = rv.Item1;
                            litText = rv.Item2;
//              nonWordPat = rv.Item3;  // the non word pattern immed after quoted literal
//              nonWordIx = rv.Item4;   // pos of foundPat.
                            nonWord   = rv.Item3;
                            tokenCode = ScanAtomCode.Quoted;
                        }

#if skip
                        // got a quote char. Isolate the quoted string, then find the delim that follows
                        // the quoted string.
                        else if ((nonWordPat.DelimClassification == DelimClassification.Quote) &&
                                 (nonWordIx == bx))
                        {
                            var rv = Scanner.IsolateQuotedWord(ScanStream, Traits, nonWordIx);
                            litType    = rv.Item1;
                            litText    = rv.Item2;
                            nonWordPat = rv.Item3; // the non word pattern immed after quoted literal
                            nonWordIx  = rv.Item4; // pos of foundPat.
                            nonWord    = rv.Item5;
                            tokenCode  = ScanAtomCode.Quoted;
                        }
#endif
                        // delim pattern found past the start of the scan. That means there are
                        // identifier chars from the start of the scan to the found delim.
                        else if (bx != nonWord.Position)
                        //            else if (bx != nonWordIx)
                        {
                            tokenCode = ScanAtomCode.Identifier;
                            tokenLx   = nonWord.Position - bx;
                            //              tokenLx = nonWordIx - bx;
                        }

                        else if (nonWordPat != null)
                        {
                            tokenCode = nonWordPat.DelimClassification.ToScanAtomCode();
                        }

                        // should never get here.
                        else
                        {
                            tokenCode = null;
                        }
                    }

                    // attempt to classify the identifier token as a keyword.
                    if ((tokenCode != null) && (tokenCode.Value == ScanAtomCode.Identifier))
                    {
                        var rv = Traits.KeywordPatterns.MatchPatternToSubstring(
                            ScanStream.Stream, bx, tokenLx.Value);
                        var kwdResults = rv.Item3;
                        var kwdPat     = kwdResults.FirstFoundPattern;
                        if (kwdPat != null)
                        {
                            tokenCode  = kwdPat.MatchPattern.DelimClassification.ToScanAtomCode();
                            nonWordPat = kwdPat.MatchPattern;
                            nonWord    = kwdResults;
                            nonWordIx  = bx;
                            nonWordLx  = kwdPat.MatchLength;
                        }

#if skip
                        var matchPat      = rv.Item1;
                        var keywordTextLx = rv.Item2; // the actual lgth of matched text.
                        if (matchPat != null)
                        {
                            tokenCode      = matchPat.DelimClassification.ToScanAtomCode();
                            nonWordPat     = matchPat;
                            nonWordPatList = null;
                            nonWord        = null;
                            nonWordIx      = bx;
                            nonWordLx      = keywordTextLx;
                        }
#endif
                    }
                }
                // end STEP 2.
                #endregion

                #region STEP 3 - setup wordBx and wordPart with the found word.
                {
                    // got whitespace.
                    if (whitespaceText != null)
                    {
                        ScanAtomCode wstc = ScanAtomCode.Whitespace;

                        if (priorTokenCode == null)
                        {
                            wstc = ScanAtomCode.InsignificantWhitespace;
                        }
                        else if (tokenCode == null)
                        {
                            wstc = ScanAtomCode.InsignificantWhitespace;
                        }
                        else if ((priorTokenCode.Value.WhitespaceIsSignificant() == true) &&
                                 (tokenCode.Value.WhitespaceIsSignificant() == true))
                        {
                            wstc = ScanAtomCode.Whitespace;
                        }
                        else
                        {
                            wstc = ScanAtomCode.InsignificantWhitespace;
                        }

                        atomText          = whitespaceText;
                        atomText.AtomCode = wstc;
                    }

                    // got nothing.
                    else if (bx == -1)
                    {
                    }

                    // no delim found. word text all the way to the end.
                    else if (nonWord.IsEmpty == true)
                    //          else if (nonWordIx == -1)
                    {
                        if (whitespaceText != null)
                        {
                            atomText   = whitespaceText;
                            nonWordPat = null;
                        }

                        else
                        {
                            // get the text from start of scan to end of string.
                            var rv = Scanner.IsolateWordText(
                                ScanStream, Traits, litType, litText, bx, null);
                            atomText = rv.Item3;
                            wordBx   = atomText.StartLoc;
                        }
                    }

                    // got a word and a non word pattern.
                    else if (nonWord.Position > bx)
                    //          else if (nonWordIx > bx)
                    {
                        if (whitespaceText != null)
                        {
                            atomText   = whitespaceText;
                            nonWord    = new PatternScanResults();
                            nonWordPat = null;
                        }

                        else
                        {
                            var rv = Scanner.IsolateWordText(
                                ScanStream, Traits, litType, litText, bx, nonWord.Position);
                            //              var rv = Scanner.IsolateWordText(
                            //                ScanStream, Traits, litType, litText, bx, nonWordIx);
                            atomText = rv.Item3;
                            wordBx   = atomText.StartLoc;
                        }
                    }

                    // no word. just delim.
                    else
                    {
                        // the delim is comment to end. store as a word.
                        if (nonWordPat.DelimClassification == DelimClassification.CommentToEnd)
                        {
                            var rv = ScanAtom.ClassifyAsComment(ScanStream, Traits, bx);
                            wordBx     = rv.Item1;
                            atomText   = rv.Item2;
                            nonWordPat = rv.Item3;
                            nonWord    = rv.Item4;
#if skip
                            var rv     = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.NewLinePatterns);
                            var eolPat = rv.Item1;
                            var eolIx  = rv.Item2;

                            // no newline pattern found. Comment to the end of the text stream.
                            if (eolPat == null)
                            {
                                int ex = ScanStream.Stream.Length - 1;
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                TextLocation wordEx      = new StreamLocation(ex).ToTextLocation(ScanStream);
                                string       commentText = ScanStream.Substring(nonWordIx);

                                string userCode = null;
                                atomText = new AtomText(
                                    ScanAtomCode.CommentToEnd, commentText, null, wordBx, wordEx,
                                    userCode);

                                nonWordPat     = null;
                                nonWordPatList = null;
                            }

                            else
                            {
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                int          lx     = eolIx - nonWordIx;
                                TextLocation wordEx =
                                    new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                string commentText = ScanStream.Substring(nonWordIx, lx);
                                string userCode    = null;
                                atomText = new AtomText(
                                    ScanAtomCode.CommentToEnd, commentText, null, wordBx, wordEx,
                                    userCode);
                                var sloc = wordBx.ToStreamLocation(ScanStream);

                                nonWordPat     = eolPat;
                                nonWordPatList = null;
                            }
#endif
                        }

                        // the word found is a non word or keyword pattern.
                        else
                        {
                            // got whitespace followed by keyword. Return the whitespace.
                            if ((nonWordPat.DelimClassification == DelimClassification.Keyword) &&
                                (whitespaceText != null))
                            {
                                atomText   = whitespaceText;
                                nonWord    = new PatternScanResults();
                                nonWordPat = null;
                            }

                            // there are more than one scan patterns that match.
                            else if (nonWord.FoundCount > 1)
                            //              else if (nonWordPatList != null)
                            {
                                atomTextList = new List <MatchScanPattern>();

                                foreach (var pat in nonWord)
                                {
                                    wordBx = new StreamLocation(nonWord.Position).ToTextLocation(ScanStream);
                                    int          lx     = pat.MatchLength;
                                    TextLocation wordEx =
                                        new StreamLocation(nonWord.Position + lx - 1).ToTextLocation(ScanStream);
                                    string scanText = ScanStream.Stream.Substring(nonWord.Position, lx);

                                    atomText = new AtomText(
                                        pat.MatchPattern.DelimClassification.ToScanAtomCode().Value,
                                        scanText,
                                        pat.MatchPattern.ReplacementValue,
                                        wordBx, wordEx,
                                        pat.MatchPattern.UserCode);

                                    pat.AtomText = atomText;
                                    atomTextList.Add(pat);
                                }
#if skip
                                foreach (var pat in nonWordPatList)
                                {
                                    wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                    int          lx     = pat.MatchLength;
                                    TextLocation wordEx =
                                        new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                    string scanText = ScanStream.Stream.Substring(nonWordIx, lx);

                                    atomText = new AtomText(
                                        pat.MatchPattern.DelimClassification.ToScanAtomCode().Value,
                                        scanText,
                                        pat.MatchPattern.ReplacementValue,
                                        wordBx, wordEx,
                                        pat.MatchPattern.UserCode);

                                    pat.AtomText = atomText;
                                    atomTextList.Add(pat);
                                }
#endif
                            }

                            else
                            {
                                var foundPat = nonWord.FirstFoundPattern;
                                wordBx = new StreamLocation(nonWord.Position).ToTextLocation(ScanStream);
                                int          lx     = foundPat.MatchLength;
                                TextLocation wordEx =
                                    new StreamLocation(nonWord.Position + lx - 1).ToTextLocation(ScanStream);
                                string scanText = ScanStream.Stream.Substring(nonWord.Position, lx);

                                atomText = new AtomText(
                                    foundPat.MatchPattern.DelimClassification.ToScanAtomCode().Value,
                                    scanText, foundPat.MatchPattern.ReplacementValue,
                                    wordBx, wordEx,
                                    foundPat.MatchPattern.UserCode);

#if skip
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                int          lx     = nonWordLx;
                                TextLocation wordEx =
                                    new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                string scanText = ScanStream.Stream.Substring(nonWordIx, lx);

                                atomText = new AtomText(
                                    nonWordPat.DelimClassification.ToScanAtomCode().Value,
                                    scanText, nonWordPat.ReplacementValue,
                                    wordBx, wordEx,
                                    nonWordPat.UserCode);
#endif
                            }
                        }
                    }
                }
                #endregion
            }

            // store the results in the return cursor.
            ScanAtomCursor nx = null;
            if (atomText == null)
            {
                nx          = new ScanAtomCursor();
                nx.Position = RelativePosition.End;
            }
            else if (atomTextList != null)
            {
                nx = new ScanAtomCursor(atomTextList);
            }
            else
            {
                //        nx = new ScanAtomCursor(atomText, nonWordPat);
                if ((nonWord == null) || (nonWord.IsEmpty == true))
                {
                    nx = new ScanAtomCursor(atomText, nonWordPat);
                }
                else
                {
                    nx = new ScanAtomCursor(atomText, nonWord.FirstFoundPattern.MatchPattern);
                }
                nx.Position = RelativePosition.At;
            }

            return(nx);
        }