internal LexToken Extract()
{
LexToken token = (LexToken)tokens[0];
tokens.RemoveAt(0);
return(token);
}
/// <summary>
/// Breaks the input stream onto the tokens list and returns it.
/// </summary>
/// <returns>The tokens list.</returns>
internal LexTokenList Lex()
{
LexToken token = GetNextToken();
while (true)
{
if (token != null)
{
tokens.Add(token);
}
else
{
tokens.Add(new LexToken(null, LexTokenTypes.EOF));
return(tokens);
}
token = GetNextToken();
}
}
internal void Push(LexToken token)
{
tokens.Insert(0, token);
}
internal void Add(LexToken token)
{
tokens.Add(token);
}
internal void Push(LexToken token)
{
tokens.Insert(0, token);
}
internal void Add(LexToken token)
{
tokens.Add(token);
}