示例#1
0
        internal LexToken Extract()
        {
            LexToken token = (LexToken)tokens[0];

            tokens.RemoveAt(0);
            return(token);
        }
示例#2
0
        /// <summary>
        /// Breaks the input stream onto the tokens list and returns it.
        /// </summary>
        /// <returns>The tokens list.</returns>
        internal LexTokenList Lex()
        {
            LexToken token = GetNextToken();

            while (true)
            {
                if (token != null)
                {
                    tokens.Add(token);
                }
                else
                {
                    tokens.Add(new LexToken(null, LexTokenTypes.EOF));
                    return(tokens);
                }

                token = GetNextToken();
            }
        }
示例#3
0
 internal void Push(LexToken token)
 {
     tokens.Insert(0, token);
 }
示例#4
0
 internal void Add(LexToken token)
 {
     tokens.Add(token);
 }
示例#5
0
 internal void Push(LexToken token)
 {
     tokens.Insert(0, token);
 }
示例#6
0
 internal void Add(LexToken token)
 {
     tokens.Add(token);
 }