private static List <Token> GetAllTokens(string code)
{
StringTokenizer.StringTokenizer stringTokenizer = new StringTokenizer.StringTokenizer(code);
List <Token> tokens = new List <Token>();
Token token;
do
{
token = stringTokenizer.Next();
tokens.Add(token);
} while (token.Kind != TokenKind.Eof);
return(tokens);
}
private static List<Token> GetAllTokens(string code)
{
StringTokenizer.StringTokenizer stringTokenizer = new StringTokenizer.StringTokenizer(code);
List<Token> tokens = new List<Token>();
Token token;
do
{
token = stringTokenizer.Next();
tokens.Add(token);
} while (token.Kind != TokenKind.Eof);
return tokens;
}
