Example #1
0
            /// <summary>
            /// Reads the next token.
            /// </summary>
            /// <returns>Read token.</returns>
            public Token ReadToken()
            {
                //The first state is almost always #1.
                int currentDFA      = m_dfaInitialState;
                int charShift       = 0;            // Next byte in the input CharReader
                int lastAcceptState = -1;           // We have not yet accepted a character string
                int acceptedChars   = 0;

                char ch = m_charReader.CurrentChar;

                if (ch != CharReader.EndOfString)
                {
                    while (true)
                    {
                        ch = m_charReader.GetChar(charShift);
                        int target = m_dfaStateTable[currentDFA].GetNextState(ch);

                        // This block-if statement checks whether an edge was found from the current state.
                        // If so, the state and current position advance. Otherwise it is time to exit the main loop
                        // and report the token found (if there was it fact one). If the LastAcceptState is -1,
                        // then we never found a match and the Error Token is created. Otherwise, a new token
                        // is created using the Symbol in the Accept State and all the characters that
                        // comprise it.
                        if (target >= 0)
                        {
                            // This code checks whether the target state accepts a token. If so, it sets the
                            // appropiate variables so when the algorithm in done, it can return the proper
                            // token and number of characters.
                            if (m_dfaStateTable[target].AcceptSymbol != null)
                            {
                                lastAcceptState = target;
                                acceptedChars   = charShift + 1;
                            }
                            currentDFA = target;
                            charShift++;
                        }
                        else
                        {
                            //No edge found
                            if (lastAcceptState == -1)
                            {
                                //Tokenizer cannot recognize symbol
                                return(new Token(m_errorSymbol, m_charReader.Accept(), LineNumber));
                            }
                            else
                            {
                                //Create Token, read characters
                                Symbol tokenSymbol = m_dfaStateTable[lastAcceptState].AcceptSymbol;
                                //The data contains the total number of accept characters
                                string data = m_charReader.Accept(acceptedChars);
                                return(new Token(tokenSymbol, data, LineNumber));
                            }
                        }
                    }
                }
                else
                {
                    return(new Token(m_endSymbol, "", LineNumber));
                }
            }