Example #1
0
        public TToken Get()
        {
            TToken token;
            skipWhiteSpace();

            //--Determine the token class, based on the current character.
            switch (CharCodeMapManager.Instance.GetCharacterMap()[buffer.CurrentChar])
            {
                case CharCode.Letter: token = new WordToken(); break;
                case CharCode.Digit: token = new NumberToken(); break;
                case CharCode.Quote: token = new StringToken(); break;
                case CharCode.Special: token = new SpecialToken(); break;
                case CharCode.EndOfFile: token = new EOFToken(); break;
                default: if (buffer.CurrentPosition == 0 && buffer.CurrentChar == 0) token = new EOFToken(); else token = new ErrorToken(); break;
            }

            //--Extract a token of that class, and return a pointer to it.
            token.Get(buffer);
            return token;
        }
        public TToken Get()
        {
            TToken pToken = null;
            TTokenCode token;
            int code;

            //--Loop to process any line markers
            //--and extract the next token code.
            do
            {
                //--First read the token code.
                code = m_formatter.ReadInt32();
                token = (TTokenCode)code;

                if (token == TTokenCode.LineMarker)
                {
                    int number = m_formatter.ReadInt32();
                    m_currentLineNumber = number;
                }

            } while (token == TTokenCode.LineMarker);

            //--Determine the token class, based on the token code.
            switch (token)
            {
                case TTokenCode.Number: pToken = new NumberToken(); break;
                case TTokenCode.String: pToken = new StringToken(); break;

                case TTokenCode.Identifier:
                    pToken = new WordToken();
                    pToken.Code = TTokenCode.Identifier;
                    break;

                default:
                    if (code < (int)TTokenCode.And)
                    {
                        pToken = new SpecialToken();
                        pToken.Code = token;
                    }
                    else
                    {
                        pToken = new WordToken(); // reserved word
                        pToken.Code = token;
                    }
                    break;
            }

            // now get node
            //--Extract the symbol table node and set the token string.
            switch (token)
            {
                case TTokenCode.Identifier:
                case TTokenCode.Number:
                case TTokenCode.String:
                    pNode = GetNode();
                    pToken.TokenString = pNode.String;
                    break;

                default:
                    pNode = null;
                    pToken.TokenString = symbolStrings[code];
                    break;
            }

            return pToken;
        }