Esempio n. 1
0
        internal virtual IEnumerable <Token> Tokenize(CharReader reader)
        {
            Compile();

            if (dfa.Start == null)
            {
                yield break;
            }

            PositionCounter start = new PositionCounter();
            PositionCounter end   = null;

            int[] tokenTypeIDs = null;

            DFA.State current = dfa.Start;

            while (!reader.IsEnd)
            {
                char c = reader.Read();
                current = current[c];

                // reached the end, nowhere else to go
                // return the match until the prev final state
                // and go back to the next char right after the prev final state
                if (current == null)
                {
                    yield return(CreateToken(reader, start, end, tokenTypeIDs));

                    reader.MoveBack(end.Position);
                    reader.Release();

                    start        = reader.PositionCounter;
                    end          = null;
                    tokenTypeIDs = null;

                    current = dfa.Start;
                    continue;
                }

                // remember this position in case we need to come back
                if (current.IsFinal)
                {
                    end          = reader.PositionCounter;
                    tokenTypeIDs = current.Values;
                }
            }

            if (end != null)
            {
                yield return(CreateToken(reader, start, end, tokenTypeIDs));

                if (end.Position != reader.Position)
                {
                    reader.MoveBack(end.Position - 1);
                    foreach (Token token in Tokenize(reader))
                    {
                        yield return(token);
                    }
                }
            }
            else
            {
                yield return(CreateToken(reader, start, reader.PositionCounter, tokenTypeIDs));
            }
        }