示例#1
0
        public static ScanAtomCursor PositionBegin()
        {
            ScanAtomCursor cursor = new ScanAtomCursor();

            cursor.Position = RelativePosition.Begin;
            return(cursor);
        }
示例#2
0
 public ScanAtomCursor(ScanAtomCursor Cursor)
 {
     this.AtomText     = Cursor.AtomText;
     this.Position     = Cursor.Position;
     this.AtomPattern  = Cursor.AtomPattern;
     this.StayAtFlag   = Cursor.StayAtFlag;
     this.ManyAtomText = Cursor.ManyAtomText;
 }
        // ----------------------- CalcScanNextStart ---------------------------
        // calc start position from which to start scan to the next word.
        private static int CalcScanNextStart(
            ScanStream ScanStream,
            TextTraits Traits, ScanAtomCursor Cursor)
        {
            int bx;

            switch (Cursor.Position)
            {
            case RelativePosition.Begin:
                bx = 0;
                break;

            case RelativePosition.Before:
                bx = Cursor.StartLoc.ToStreamLocation(ScanStream).Value;
                break;

            case RelativePosition.After:
            case RelativePosition.At:
                bx = Cursor.EndLoc.ToStreamLocation(ScanStream).Value + 1;
                break;

            case RelativePosition.End:
                bx = ScanStream.Stream.Length;
                break;

            case RelativePosition.None:
                bx = -1;
                break;

            default:
                bx = -1;
                break;
            }

            if (bx > (ScanStream.Stream.Length - 1))
            {
                bx = -1;
            }

            return(bx);
        }
        // ------------------------ ScanNextAtom -------------------------
        // Scans to the next atom in the string. ( a word being the text bounded by the
        // delimeter and whitespace characters as spcfd in the TextTraits argument )
        // Return null when end of string.
        public static ScanAtomCursor ScanNextAtom(
            ScanStream ScanStream,
            TextTraits Traits, ScanAtomCursor CurrentWord)
        {
            // components of the next word.
            TextLocation wordBx    = null;
            int          nonWordIx = -1;
            int          nonWordLx = 0;

            ScanPattern             nonWordPat     = null;
            List <MatchScanPattern> nonWordPatList = null;
            AtomText atomText = null;
            List <MatchScanPattern> atomTextList = null;
            AtomText whitespaceText = null;
//      ScanAtomCode? priorCode = null;
            bool?priorCodeIsWhitespaceSignificant = null;

            // stay at the current location. return copy of the cursor, but with stayatflag
            // turned off.
            if (CurrentWord.StayAtFlag == true)
            {
                atomText   = CurrentWord.AtomText;
                nonWordPat = CurrentWord.AtomPattern;
                wordBx     = CurrentWord.StartLoc;
            }

            else
            {
                #region STEP1 setup the begin pos of the next word.
                // ----------------------------- STEP 1 ------------------------------
                // setup the begin pos of the next word.
                int bx;
                {
                    // save the ScanAtomCode of the prior word.
                    if ((CurrentWord.Position == RelativePosition.At) ||
                        (CurrentWord.Position == RelativePosition.After))
                    {
                        priorCodeIsWhitespaceSignificant = CurrentWord.WhitespaceIsSignificant;
//            priorCode = CurrentWord.AtomText.AtomCode;
                    }

                    // calc scan start position
                    bx = ScanAtom.CalcScanNextStart(ScanStream, Traits, CurrentWord);

                    // advance past whitespace
                    if (bx != -1)
                    {
                        int saveBx = bx;
                        bx = Scanner.ScanNotEqual(ScanStream.Stream, Traits.WhitespacePatterns, bx);

                        // there is some whitespace. depending on what preceeds and follows, may
                        // return this as the atom.
                        if ((priorCodeIsWhitespaceSignificant != null) &&
                            (priorCodeIsWhitespaceSignificant.Value == true))
                        {
                            if (bx != saveBx)
                            {
                                int whitespaceEx = -1;
                                if (bx == -1)
                                {
                                    whitespaceEx = ScanStream.Stream.Length - 1;
                                }
                                else
                                {
                                    whitespaceEx = bx - 1;
                                }
                                int whitespaceLx = whitespaceEx - saveBx + 1;

                                whitespaceText = new AtomText(
                                    ScanAtomCode.Whitespace,
                                    ScanStream.Stream.Substring(saveBx, whitespaceLx), " ",
                                    new StreamLocation(saveBx).ToTextLocation(ScanStream),
                                    new StreamLocation(whitespaceEx).ToTextLocation(ScanStream));
                            }
                        }
                    }
                }
                // end STEP 1.
                #endregion

                #region STEP 2. Isolate either numeric lib, quoted lit or scan to non word pattern
                // ------------------------------- STEP 2 ----------------------------------
                // Isolate either numeric literal, quoted literal or scan to the next non word
                // pattern.
                LiteralType?litType = null;
                string      litText = null;
                {
                    // got a decimal digit. isolate the numeric literal string.
                    if ((bx != -1) && (Char.IsDigit(ScanStream.Stream[bx]) == true))
                    {
                        var rv = Scanner.IsolateNumericLiteral(ScanStream, Traits, bx);
                        litType    = rv.Item1;
                        litText    = rv.Item2;
                        nonWordPat = rv.Item3; // the non word pattern immed after numeric literal
                        nonWordIx  = rv.Item4; // pos of foundPat
                    }

                    // got something.  now scan forward for the pattern that delimits the word.
                    else if (bx != -1)
                    {
                        {
                            var rv = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.DelimPatterns);
                            nonWordPat     = rv.Item1;
                            nonWordIx      = rv.Item2;
                            nonWordLx      = rv.Item3;
                            nonWordPatList = rv.Item4;
                        }

                        // got a quote char. Isolate the quoted string, then find the delim that follows
                        // the quoted string.
                        if ((nonWordPat != null) &&
                            (nonWordPat.DelimClassification == DelimClassification.Quote) &&
                            (nonWordIx == bx))
                        {
                            var rv = Scanner.IsolateQuotedWord(ScanStream, Traits, nonWordIx);
                            litType    = rv.Item1;
                            litText    = rv.Item2;
                            nonWordPat = rv.Item3; // the non word pattern immed after quoted literal
                            nonWordIx  = rv.Item4; // pos of foundPat.
                        }
                    }
                }
                // end STEP 2.
                #endregion

                #region STEP 3 - setup wordBx and wordPart with the found word.
                {
                    // got nothing.
                    if (bx == -1)
                    {
                    }

                    // no delim found. word text all the way to the end.
                    else if (nonWordIx == -1)
                    {
                        if (whitespaceText != null)
                        {
                            atomText       = whitespaceText;
                            nonWordPat     = null;
                            nonWordPatList = null;
                        }

                        else
                        {
                            var rv = Scanner.IsolateWordText(
                                ScanStream, Traits, litType, litText, bx, null);
                            atomText = rv.Item3;
                            wordBx   = atomText.StartLoc;
                        }
                    }

                    // got a word and a non word pattern.
                    else if (nonWordIx > bx)
                    {
                        if (whitespaceText != null)
                        {
                            atomText       = whitespaceText;
                            nonWordPat     = null;
                            nonWordPatList = null;
                        }

                        else
                        {
                            var rv = Scanner.IsolateWordText(
                                ScanStream, Traits, litType, litText, bx, nonWordIx);
                            atomText = rv.Item3;
                            wordBx   = atomText.StartLoc;
                        }
                    }

                    // no word. just delim.
                    else
                    {
                        // the delim is comment to end. store as a word.
                        if (nonWordPat.DelimClassification == DelimClassification.CommentToEnd)
                        {
                            var rv     = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.NewLinePatterns);
                            var eolPat = rv.Item1;
                            var eolIx  = rv.Item2;
                            if (eolPat == null)
                            {
                                int ex = ScanStream.Stream.Length - 1;
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                TextLocation wordEx      = new StreamLocation(ex).ToTextLocation(ScanStream);
                                string       commentText = ScanStream.Substring(nonWordIx);

                                atomText = new AtomText(
                                    ScanAtomCode.CommentToEnd, commentText, null, wordBx, wordEx);

                                nonWordPat     = null;
                                nonWordPatList = null;
                            }
                            else
                            {
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                int          lx     = eolIx - nonWordIx;
                                TextLocation wordEx =
                                    new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                string commentText = ScanStream.Substring(nonWordIx, lx);
                                atomText = new AtomText(
                                    ScanAtomCode.CommentToEnd, commentText, null, wordBx, wordEx);
                                var sloc = wordBx.ToStreamLocation(ScanStream);

                                nonWordPat     = eolPat;
                                nonWordPatList = null;
                            }
                        }

                        // the word found is a non word or keyword pattern.
                        else
                        {
                            // got whitespace followed by keyword. Return the whitespace.
                            if ((nonWordPat.DelimClassification == DelimClassification.Keyword) &&
                                (whitespaceText != null))
                            {
                                atomText       = whitespaceText;
                                nonWordPat     = null;
                                nonWordPatList = null;
                            }

                            // there are more than one scan patterns that match.
                            else if (nonWordPatList != null)
                            {
                                atomTextList = new List <MatchScanPattern>();
                                foreach (var pat in nonWordPatList)
                                {
                                    wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                    int          lx     = pat.MatchLength;
                                    TextLocation wordEx =
                                        new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                    string scanText = ScanStream.Stream.Substring(nonWordIx, lx);

                                    atomText = new AtomText(
                                        pat.MatchPattern.DelimClassification.ToScanAtomCode().Value,
                                        scanText,
                                        pat.MatchPattern.ReplacementValue,
                                        wordBx, wordEx);

                                    pat.AtomText = atomText;
                                    atomTextList.Add(pat);
                                }
                            }

                            else
                            {
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                int          lx     = nonWordLx;
                                TextLocation wordEx =
                                    new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                string scanText = ScanStream.Stream.Substring(nonWordIx, lx);

                                atomText = new AtomText(
                                    nonWordPat.DelimClassification.ToScanAtomCode().Value,
                                    scanText, nonWordPat.ReplacementValue,
                                    wordBx, wordEx);
                            }
                        }
                    }
                }
                #endregion
            }

            // store the results in the return cursor.
            ScanAtomCursor nx = null;
            if (atomText == null)
            {
                nx          = new ScanAtomCursor( );
                nx.Position = RelativePosition.End;
            }
            else if (atomTextList != null)
            {
                nx = new ScanAtomCursor(atomTextList);
            }
            else
            {
                nx          = new ScanAtomCursor(atomText, nonWordPat);
                nx.Position = RelativePosition.At;
            }

            return(nx);
        }
        // ------------------------ ScanNextAtom -------------------------
        // Scans to the next atom in the string. ( a word being the text bounded by the
        // delimiter and whitespace characters as spcfd in the TextTraits argument )
        // Return null when end of string.
        public static ScanAtomCursor ScanNextAtom(
            ScanStream ScanStream,
            TextTraits Traits, ScanAtomCursor CurrentWord)
        {
            PatternScanResults nonWord = null;

            AtomText atomText = null;
            List <MatchScanPattern> atomTextList = null;
            AtomText whitespaceText = null;

            ScanAtomCode?tokenCode      = null; // ScanAtomCode of this token.
            int?         tokenLx        = null;
            ScanAtomCode?priorTokenCode = null;

            bool?priorCodeIsWhitespaceSignificant = null;

            // stay at the current location. return copy of the cursor, but with stayatflag
            // turned off.
            if (CurrentWord.StayAtFlag == true)
            {
                atomText       = CurrentWord.AtomText;
                tokenCode      = atomText.AtomCode;
                priorTokenCode = null;
                nonWord        = new PatternScanResults(
                    CurrentWord.AtomPattern,
                    CurrentWord.StartLoc.ToStreamLocation(ScanStream).Value,
                    CurrentWord.AtomPattern.Length);
            }

            else
            {
                #region STEP1 setup the begin pos of the next word.
                // ----------------------------- STEP 1 ------------------------------
                // setup the begin pos of the next word.
                int bx;
                {
                    // save the ScanAtomCode of the prior word.
                    if ((CurrentWord.Position == RelativePosition.At) ||
                        (CurrentWord.Position == RelativePosition.After))
                    {
                        priorTokenCode = CurrentWord.AtomCode;
                        priorCodeIsWhitespaceSignificant = CurrentWord.WhitespaceIsSignificant;
                    }

                    // calc scan start position
                    bx = ScanAtom.CalcScanNextStart(ScanStream, Traits, CurrentWord);

                    // advance past whitespace
                    if (bx != -1)
                    {
                        int saveBx = bx;
                        bx = Scanner.ScanNotEqual(ScanStream.Stream, Traits.WhitespacePatterns, bx);

                        // there is some whitespace. Isolate it as AtomText.
                        // This method will return the whitespace as the token. But need to look at
                        // the token before and after to classify the whitespace as significant or
                        // not. ( whitespace between identifiers or keywords is significant.
                        // Whitespace between symbols is not significant.
                        // note: even insignificant whitespace is returned as a token because the
                        //       whitespace is needed when redisplaying the statement text.
                        if (bx != saveBx)
                        {
                            int whitespaceEx = -1;
                            if (bx == -1)
                            {
                                whitespaceEx = ScanStream.Stream.Length - 1;
                            }
                            else
                            {
                                whitespaceEx = bx - 1;
                            }
                            int whitespaceLx = whitespaceEx - saveBx + 1;

                            // split the whitespace between space/tab and EOL
                            {
                                int fx1 = ScanStream.Stream.IndexOfAny(new char[] { ' ', '\t' }, saveBx);
                                int fx2 = ScanStream.Stream.IndexOfAny(new char[] { '\r', '\n' }, saveBx);
                                if (fx1 > whitespaceEx)
                                {
                                    fx1 = -1;
                                }
                                if (fx2 > whitespaceEx)
                                {
                                    fx2 = -1;
                                }
                                if ((fx1 == saveBx) && (fx2 != -1))
                                {
                                    whitespaceEx = fx2 - 1;
                                }
                                if ((fx2 == saveBx) && (fx1 != -1))
                                {
                                    whitespaceEx = fx1 - 1;
                                }
                                whitespaceLx = whitespaceEx - saveBx + 1;
                            }

                            string userCode = null;
                            whitespaceText = new AtomText(
                                ScanAtomCode.Whitespace,
                                ScanStream.Stream.Substring(saveBx, whitespaceLx), " ",
                                new StreamLocation(saveBx).ToTextLocation(ScanStream),
                                new StreamLocation(whitespaceEx).ToTextLocation(ScanStream),
                                userCode);
                        }
                    }
                }
                // end STEP 1.
                #endregion

                #region STEP 2. Isolate either numeric lit, quoted lit or identifier/keyword.
                // ------------------------------- STEP 2 ----------------------------------
                // Isolate either numeric literal, quoted literal or scan to the next non word
                // pattern.
                LiteralType?litType = null;
                string      litText = null;
                {
                    // got a decimal digit. isolate the numeric literal string.
                    if ((bx != -1) && (Char.IsDigit(ScanStream.Stream[bx]) == true))
                    {
                        var rv = Scanner.IsolateNumericLiteral(ScanStream, Traits, bx);
                        litType   = rv.Item1;
                        litText   = rv.Item2;
                        nonWord   = rv.Item3; // the non word pattern immed after numeric literal
                        tokenCode = ScanAtomCode.Numeric;
                    }

                    // got something.  now scan forward for the pattern that delimits the word.
                    else if (bx != -1)
                    {
                        {
                            nonWord = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.DelimPatterns);
                        }

                        // a special value starter. scan further for the spcval word.
                        // If an identifier follows
                        var startPat = nonWord.FindPattern(DelimClassification.SpecialValueStarter);
                        if (startPat != null)
                        {
                            var csr = new ScanAtomCursor(startPat, ScanStream);
                            var nx  = ScanAtom.ScanNextAtom(ScanStream, Traits, csr);
                            if ((nx.Position == RelativePosition.At) &&
                                (nx.AtomCode.IsIdentifier() == true))
                            {
                                atomText = AtomText.Combine(
                                    startPat.AtomText, nx.AtomText, ScanAtomCode.SpecialValue);
                            }
                        }

                        // got the AtomText of the token.
                        if (atomText != null)
                        {
                        }

                        // word chars all the way to the end.
                        else if (nonWord == null)
                        {
                            tokenCode = ScanAtomCode.Identifier;
                            tokenLx   = ScanStream.Stream.Length - bx;
                        }

                        else if (nonWord.FoundAtPosition(DelimClassification.Quote, bx))
                        {
                            var rv = Scanner.IsolateQuotedWord(ScanStream, Traits, bx);
                            litType   = rv.Item1;
                            litText   = rv.Item2;
                            nonWord   = rv.Item3; // the non word pattern immed after quoted literal
                            tokenCode = ScanAtomCode.Quoted;
                        }

                        // delim pattern found past the start of the scan. That means there are
                        // identifier chars from the start of the scan to the found delim.
                        else if (bx != nonWord.Position)
                        {
                            tokenCode = ScanAtomCode.Identifier;
                            tokenLx   = nonWord.Position - bx;
                        }

                        else if (nonWord.IsEmpty == false)
                        {
                            tokenCode =
                                nonWord.FirstFoundPattern.MatchPattern.DelimClassification.ToScanAtomCode();
                        }

                        // should never get here.
                        else
                        {
                            tokenCode = null;
                        }
                    }

                    // attempt to classify the identifier token as a keyword.
                    if (atomText == null)
                    {
                        if ((tokenCode != null) && (tokenCode.Value == ScanAtomCode.Identifier))
                        {
                            var rv = Traits.KeywordPatterns.MatchPatternToSubstring(
                                ScanStream.Stream, bx, tokenLx.Value);
                            var kwdResults = rv.Item3;
                            var kwdPat     = kwdResults.FirstFoundPattern;
                            if (kwdPat != null)
                            {
                                tokenCode = kwdPat.MatchPattern.DelimClassification.ToScanAtomCode();
                                nonWord   = kwdResults;
                            }
                        }
                    }
                }
                // end STEP 2.
                #endregion

                #region STEP 3 - setup atomText of the found token.
                {
                    // got the atomText of the token.
                    if (atomText != null)
                    {
                        nonWord = null;
                    }

                    // got whitespace.
                    else if (whitespaceText != null)
                    {
                        ScanAtomCode wstc = ScanAtomCode.Whitespace;

                        if (priorTokenCode == null)
                        {
                            wstc = ScanAtomCode.InsignificantWhitespace;
                        }
                        else if (tokenCode == null)
                        {
                            wstc = ScanAtomCode.InsignificantWhitespace;
                        }
                        else if ((priorTokenCode.Value.WhitespaceIsSignificant() == true) &&
                                 (tokenCode.Value.WhitespaceIsSignificant() == true))
                        {
                            wstc = ScanAtomCode.Whitespace;
                        }
                        else
                        {
                            wstc = ScanAtomCode.InsignificantWhitespace;
                        }

                        atomText          = whitespaceText;
                        atomText.AtomCode = wstc;
                    }

                    // got nothing.
                    else if (bx == -1)
                    {
                    }

                    // no delim found. word text all the way to the end.
                    else if (nonWord.IsEmpty == true)
                    {
                        if (whitespaceText != null)
                        {
                            atomText = whitespaceText;
                        }

                        else
                        {
                            // get the text from start of scan to end of string.
                            var rv = Scanner.IsolateWordText(
                                ScanStream, Traits, litType, litText, bx, null);
                            atomText = rv.Item3;
                        }
                    }

                    // got a word followed by non word pattern. return the word.
                    else if (nonWord.Position > bx)
                    {
                        if (whitespaceText != null)
                        {
                            atomText = whitespaceText;
                            nonWord  = new PatternScanResults();
                        }

                        else
                        {
                            var rv = Scanner.IsolateWordText(
                                ScanStream, Traits, litType, litText, bx, nonWord.Position);
                            atomText = rv.Item3;
                        }
                    }

                    // no word. just delim.
                    else
                    {
                        // the delim is comment to end. store as a word.
                        if (nonWord.FirstFoundPattern.MatchPattern.DelimClassification ==
                            DelimClassification.CommentToEnd)
                        {
                            var rv = ScanAtom.ClassifyAsComment(ScanStream, Traits, bx);
                            atomText = rv.Item2;
                            nonWord  = rv.Item4;
                        }

                        // the word found is a non word or keyword pattern.
                        else
                        {
                            // got whitespace followed by keyword. Return the whitespace.
                            if ((nonWord.FirstFoundPattern.MatchPattern.DelimClassification
                                 == DelimClassification.Keyword) &&
                                (whitespaceText != null))
                            {
                                atomText = whitespaceText;
                                nonWord  = new PatternScanResults();
                            }

                            // there are more than one scan patterns that match.
                            else if (nonWord.FoundCount > 1)
                            {
                                atomTextList = new List <MatchScanPattern>();

                                foreach (var pat in nonWord)
                                {
                                    pat.AssignAtomText(ScanStream);
                                    atomTextList.Add(pat);
                                }
                            }

                            else
                            {
                                var foundPat = nonWord.FirstFoundPattern;
                                foundPat.AssignAtomText(ScanStream);
                                atomText = foundPat.AtomText;
                            }
                        }
                    }
                }
                #endregion
            }

            // store the results in the return cursor.
            {
                ScanAtomCursor nx = null;
                if (atomText == null)
                {
                    nx          = new ScanAtomCursor();
                    nx.Position = RelativePosition.End;
                }
                else if (atomTextList != null)
                {
                    nx = new ScanAtomCursor(atomTextList);
                }
                else
                {
                    if ((nonWord == null) || (nonWord.IsEmpty == true))
                    {
                        nx = new ScanAtomCursor(atomText, null);
                    }
                    else
                    {
                        nx = new ScanAtomCursor(atomText, nonWord.FirstFoundPattern.MatchPattern);
                    }
                    nx.Position = RelativePosition.At;
                }

                return(nx);
            }
        }
        // ------------------------ ScanNextAtom -------------------------
        // Scans to the next atom in the string. ( a word being the text bounded by the
        // delimiter and whitespace characters as spcfd in the TextTraits argument )
        // Return null when end of string.
        public static ScanAtomCursor OrigScanNextAtom(
            ScanStream ScanStream,
            TextTraits Traits, ScanAtomCursor CurrentWord)
        {
            // components of the next word.
            TextLocation wordBx    = null;
            int          nonWordIx = -1;
            int          nonWordLx = 0;

            ScanPattern        nonWordPat = null;
            PatternScanResults nonWord    = null;

            AtomText atomText = null;
            List <MatchScanPattern> atomTextList = null;
            AtomText whitespaceText = null;

            ScanAtomCode?tokenCode      = null; // ScanAtomCode of this token.
            int?         tokenLx        = null;
            ScanAtomCode?priorTokenCode = null;

            //      ScanAtomCode? priorCode = null;
            bool?priorCodeIsWhitespaceSignificant = null;

            // stay at the current location. return copy of the cursor, but with stayatflag
            // turned off.
            if (CurrentWord.StayAtFlag == true)
            {
                atomText       = CurrentWord.AtomText;
                tokenCode      = atomText.AtomCode;
                priorTokenCode = null;
                nonWordPat     = CurrentWord.AtomPattern;
                wordBx         = CurrentWord.StartLoc;
            }

            else
            {
                #region STEP1 setup the begin pos of the next word.
                // ----------------------------- STEP 1 ------------------------------
                // setup the begin pos of the next word.
                int bx;
                {
                    // save the ScanAtomCode of the prior word.
                    if ((CurrentWord.Position == RelativePosition.At) ||
                        (CurrentWord.Position == RelativePosition.After))
                    {
                        priorTokenCode = CurrentWord.AtomCode;
                        priorCodeIsWhitespaceSignificant = CurrentWord.WhitespaceIsSignificant;
                    }

                    // calc scan start position
                    bx = ScanAtom.CalcScanNextStart(ScanStream, Traits, CurrentWord);

                    // advance past whitespace
                    if (bx != -1)
                    {
                        int saveBx = bx;
                        bx = Scanner.ScanNotEqual(ScanStream.Stream, Traits.WhitespacePatterns, bx);

                        // there is some whitespace. Isolate it as AtomText.
                        // This method will return the whitespace as the token. But need to look at
                        // the token before and after to classify the whitespace as significant or
                        // not. ( whitespace between identifiers or keywords is significant.
                        // Whitespace between symbols is not significant.
                        // note: even insignificant whitespace is returned as a token because the
                        //       whitespace is needed when redisplaying the statement text.
                        if (bx != saveBx)
                        {
                            int whitespaceEx = -1;
                            if (bx == -1)
                            {
                                whitespaceEx = ScanStream.Stream.Length - 1;
                            }
                            else
                            {
                                whitespaceEx = bx - 1;
                            }
                            int whitespaceLx = whitespaceEx - saveBx + 1;

                            string userCode = null;
                            whitespaceText = new AtomText(
                                ScanAtomCode.Whitespace,
                                ScanStream.Stream.Substring(saveBx, whitespaceLx), " ",
                                new StreamLocation(saveBx).ToTextLocation(ScanStream),
                                new StreamLocation(whitespaceEx).ToTextLocation(ScanStream),
                                userCode);
                        }
                    }
                }
                // end STEP 1.
                #endregion

                #region STEP 2. Isolate either numeric lit, quoted lit or identifier/keyword.
                // ------------------------------- STEP 2 ----------------------------------
                // Isolate either numeric literal, quoted literal or scan to the next non word
                // pattern.
                LiteralType?litType = null;
                string      litText = null;
                {
                    // got a decimal digit. isolate the numeric literal string.
                    if ((bx != -1) && (Char.IsDigit(ScanStream.Stream[bx]) == true))
                    {
                        var rv = Scanner.IsolateNumericLiteral(ScanStream, Traits, bx);
                        litType = rv.Item1;
                        litText = rv.Item2;
                        nonWord = rv.Item3;
//            nonWordPat = rv.Item4;  // the non word pattern immed after numeric literal
//            nonWordIx = rv.Item5;   // pos of foundPat
                        tokenCode = ScanAtomCode.Numeric;
                    }

                    // got something.  now scan forward for the pattern that delimits the word.
                    else if (bx != -1)
                    {
                        {
                            nonWord = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.DelimPatterns);
//              nonWordPat = rv.Item1;
//              nonWordIx = rv.Item2;
//              nonWordLx = rv.Item3;
//              nonWord = rv.Item3;
                        }

                        // a special value starter. scan further for the spcval word.
                        var startPat = nonWord.FindPattern(DelimClassification.SpecialValueStarter);
                        if (startPat != null)
                        {
                        }

                        // word chars all the way to the end.
                        //            if (nonWordPat == null)
                        if (nonWord == null)
                        {
                            tokenCode = ScanAtomCode.Identifier;
                            tokenLx   = ScanStream.Stream.Length - bx;
                        }

                        else if (nonWord.FoundAtPosition(DelimClassification.Quote, bx))
                        {
                            var rv = Scanner.IsolateQuotedWord(ScanStream, Traits, bx);
                            litType = rv.Item1;
                            litText = rv.Item2;
//              nonWordPat = rv.Item3;  // the non word pattern immed after quoted literal
//              nonWordIx = rv.Item4;   // pos of foundPat.
                            nonWord   = rv.Item3;
                            tokenCode = ScanAtomCode.Quoted;
                        }

#if skip
                        // got a quote char. Isolate the quoted string, then find the delim that follows
                        // the quoted string.
                        else if ((nonWordPat.DelimClassification == DelimClassification.Quote) &&
                                 (nonWordIx == bx))
                        {
                            var rv = Scanner.IsolateQuotedWord(ScanStream, Traits, nonWordIx);
                            litType    = rv.Item1;
                            litText    = rv.Item2;
                            nonWordPat = rv.Item3; // the non word pattern immed after quoted literal
                            nonWordIx  = rv.Item4; // pos of foundPat.
                            nonWord    = rv.Item5;
                            tokenCode  = ScanAtomCode.Quoted;
                        }
#endif
                        // delim pattern found past the start of the scan. That means there are
                        // identifier chars from the start of the scan to the found delim.
                        else if (bx != nonWord.Position)
                        //            else if (bx != nonWordIx)
                        {
                            tokenCode = ScanAtomCode.Identifier;
                            tokenLx   = nonWord.Position - bx;
                            //              tokenLx = nonWordIx - bx;
                        }

                        else if (nonWordPat != null)
                        {
                            tokenCode = nonWordPat.DelimClassification.ToScanAtomCode();
                        }

                        // should never get here.
                        else
                        {
                            tokenCode = null;
                        }
                    }

                    // attempt to classify the identifier token as a keyword.
                    if ((tokenCode != null) && (tokenCode.Value == ScanAtomCode.Identifier))
                    {
                        var rv = Traits.KeywordPatterns.MatchPatternToSubstring(
                            ScanStream.Stream, bx, tokenLx.Value);
                        var kwdResults = rv.Item3;
                        var kwdPat     = kwdResults.FirstFoundPattern;
                        if (kwdPat != null)
                        {
                            tokenCode  = kwdPat.MatchPattern.DelimClassification.ToScanAtomCode();
                            nonWordPat = kwdPat.MatchPattern;
                            nonWord    = kwdResults;
                            nonWordIx  = bx;
                            nonWordLx  = kwdPat.MatchLength;
                        }

#if skip
                        var matchPat      = rv.Item1;
                        var keywordTextLx = rv.Item2; // the actual lgth of matched text.
                        if (matchPat != null)
                        {
                            tokenCode      = matchPat.DelimClassification.ToScanAtomCode();
                            nonWordPat     = matchPat;
                            nonWordPatList = null;
                            nonWord        = null;
                            nonWordIx      = bx;
                            nonWordLx      = keywordTextLx;
                        }
#endif
                    }
                }
                // end STEP 2.
                #endregion

                #region STEP 3 - setup wordBx and wordPart with the found word.
                {
                    // got whitespace.
                    if (whitespaceText != null)
                    {
                        ScanAtomCode wstc = ScanAtomCode.Whitespace;

                        if (priorTokenCode == null)
                        {
                            wstc = ScanAtomCode.InsignificantWhitespace;
                        }
                        else if (tokenCode == null)
                        {
                            wstc = ScanAtomCode.InsignificantWhitespace;
                        }
                        else if ((priorTokenCode.Value.WhitespaceIsSignificant() == true) &&
                                 (tokenCode.Value.WhitespaceIsSignificant() == true))
                        {
                            wstc = ScanAtomCode.Whitespace;
                        }
                        else
                        {
                            wstc = ScanAtomCode.InsignificantWhitespace;
                        }

                        atomText          = whitespaceText;
                        atomText.AtomCode = wstc;
                    }

                    // got nothing.
                    else if (bx == -1)
                    {
                    }

                    // no delim found. word text all the way to the end.
                    else if (nonWord.IsEmpty == true)
                    //          else if (nonWordIx == -1)
                    {
                        if (whitespaceText != null)
                        {
                            atomText   = whitespaceText;
                            nonWordPat = null;
                        }

                        else
                        {
                            // get the text from start of scan to end of string.
                            var rv = Scanner.IsolateWordText(
                                ScanStream, Traits, litType, litText, bx, null);
                            atomText = rv.Item3;
                            wordBx   = atomText.StartLoc;
                        }
                    }

                    // got a word and a non word pattern.
                    else if (nonWord.Position > bx)
                    //          else if (nonWordIx > bx)
                    {
                        if (whitespaceText != null)
                        {
                            atomText   = whitespaceText;
                            nonWord    = new PatternScanResults();
                            nonWordPat = null;
                        }

                        else
                        {
                            var rv = Scanner.IsolateWordText(
                                ScanStream, Traits, litType, litText, bx, nonWord.Position);
                            //              var rv = Scanner.IsolateWordText(
                            //                ScanStream, Traits, litType, litText, bx, nonWordIx);
                            atomText = rv.Item3;
                            wordBx   = atomText.StartLoc;
                        }
                    }

                    // no word. just delim.
                    else
                    {
                        // the delim is comment to end. store as a word.
                        if (nonWordPat.DelimClassification == DelimClassification.CommentToEnd)
                        {
                            var rv = ScanAtom.ClassifyAsComment(ScanStream, Traits, bx);
                            wordBx     = rv.Item1;
                            atomText   = rv.Item2;
                            nonWordPat = rv.Item3;
                            nonWord    = rv.Item4;
#if skip
                            var rv     = Scanner.ScanEqualAny(ScanStream.Stream, bx, Traits.NewLinePatterns);
                            var eolPat = rv.Item1;
                            var eolIx  = rv.Item2;

                            // no newline pattern found. Comment to the end of the text stream.
                            if (eolPat == null)
                            {
                                int ex = ScanStream.Stream.Length - 1;
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                TextLocation wordEx      = new StreamLocation(ex).ToTextLocation(ScanStream);
                                string       commentText = ScanStream.Substring(nonWordIx);

                                string userCode = null;
                                atomText = new AtomText(
                                    ScanAtomCode.CommentToEnd, commentText, null, wordBx, wordEx,
                                    userCode);

                                nonWordPat     = null;
                                nonWordPatList = null;
                            }

                            else
                            {
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                int          lx     = eolIx - nonWordIx;
                                TextLocation wordEx =
                                    new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                string commentText = ScanStream.Substring(nonWordIx, lx);
                                string userCode    = null;
                                atomText = new AtomText(
                                    ScanAtomCode.CommentToEnd, commentText, null, wordBx, wordEx,
                                    userCode);
                                var sloc = wordBx.ToStreamLocation(ScanStream);

                                nonWordPat     = eolPat;
                                nonWordPatList = null;
                            }
#endif
                        }

                        // the word found is a non word or keyword pattern.
                        else
                        {
                            // got whitespace followed by keyword. Return the whitespace.
                            if ((nonWordPat.DelimClassification == DelimClassification.Keyword) &&
                                (whitespaceText != null))
                            {
                                atomText   = whitespaceText;
                                nonWord    = new PatternScanResults();
                                nonWordPat = null;
                            }

                            // there are more than one scan patterns that match.
                            else if (nonWord.FoundCount > 1)
                            //              else if (nonWordPatList != null)
                            {
                                atomTextList = new List <MatchScanPattern>();

                                foreach (var pat in nonWord)
                                {
                                    wordBx = new StreamLocation(nonWord.Position).ToTextLocation(ScanStream);
                                    int          lx     = pat.MatchLength;
                                    TextLocation wordEx =
                                        new StreamLocation(nonWord.Position + lx - 1).ToTextLocation(ScanStream);
                                    string scanText = ScanStream.Stream.Substring(nonWord.Position, lx);

                                    atomText = new AtomText(
                                        pat.MatchPattern.DelimClassification.ToScanAtomCode().Value,
                                        scanText,
                                        pat.MatchPattern.ReplacementValue,
                                        wordBx, wordEx,
                                        pat.MatchPattern.UserCode);

                                    pat.AtomText = atomText;
                                    atomTextList.Add(pat);
                                }
#if skip
                                foreach (var pat in nonWordPatList)
                                {
                                    wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                    int          lx     = pat.MatchLength;
                                    TextLocation wordEx =
                                        new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                    string scanText = ScanStream.Stream.Substring(nonWordIx, lx);

                                    atomText = new AtomText(
                                        pat.MatchPattern.DelimClassification.ToScanAtomCode().Value,
                                        scanText,
                                        pat.MatchPattern.ReplacementValue,
                                        wordBx, wordEx,
                                        pat.MatchPattern.UserCode);

                                    pat.AtomText = atomText;
                                    atomTextList.Add(pat);
                                }
#endif
                            }

                            else
                            {
                                var foundPat = nonWord.FirstFoundPattern;
                                wordBx = new StreamLocation(nonWord.Position).ToTextLocation(ScanStream);
                                int          lx     = foundPat.MatchLength;
                                TextLocation wordEx =
                                    new StreamLocation(nonWord.Position + lx - 1).ToTextLocation(ScanStream);
                                string scanText = ScanStream.Stream.Substring(nonWord.Position, lx);

                                atomText = new AtomText(
                                    foundPat.MatchPattern.DelimClassification.ToScanAtomCode().Value,
                                    scanText, foundPat.MatchPattern.ReplacementValue,
                                    wordBx, wordEx,
                                    foundPat.MatchPattern.UserCode);

#if skip
                                wordBx = new StreamLocation(nonWordIx).ToTextLocation(ScanStream);
                                int          lx     = nonWordLx;
                                TextLocation wordEx =
                                    new StreamLocation(nonWordIx + lx - 1).ToTextLocation(ScanStream);
                                string scanText = ScanStream.Stream.Substring(nonWordIx, lx);

                                atomText = new AtomText(
                                    nonWordPat.DelimClassification.ToScanAtomCode().Value,
                                    scanText, nonWordPat.ReplacementValue,
                                    wordBx, wordEx,
                                    nonWordPat.UserCode);
#endif
                            }
                        }
                    }
                }
                #endregion
            }

            // store the results in the return cursor.
            ScanAtomCursor nx = null;
            if (atomText == null)
            {
                nx          = new ScanAtomCursor();
                nx.Position = RelativePosition.End;
            }
            else if (atomTextList != null)
            {
                nx = new ScanAtomCursor(atomTextList);
            }
            else
            {
                //        nx = new ScanAtomCursor(atomText, nonWordPat);
                if ((nonWord == null) || (nonWord.IsEmpty == true))
                {
                    nx = new ScanAtomCursor(atomText, nonWordPat);
                }
                else
                {
                    nx = new ScanAtomCursor(atomText, nonWord.FirstFoundPattern.MatchPattern);
                }
                nx.Position = RelativePosition.At;
            }

            return(nx);
        }