IReadOnlyList<IToken> ITokenizer.GetTokens(string text)
{
List<IToken> tokens = new List<IToken>();
StringTokenizer tokenizer = new StringTokenizer(text);
tokenizer.IgnoreWhiteSpace = true;
Token current = null;
do
{
current = tokenizer.Next();
if (current.Kind == TokenKind.EOL)
{
}
IToken token = new TokenImpl()
{
Content = current.Value,
Location = new SourceLocation() { Column = current.Column, Line = current.Line },
Type = (TokenType)(int)current.Kind,
};
tokens.Add(token);
} while (current.Kind != TokenKind.EOF);
return tokens;
}
IReadOnlyList <IToken> ITokenizer.GetTokens(string text)
{
List <IToken> tokens = new List <IToken>();
StringTokenizer tokenizer = new StringTokenizer(text);
tokenizer.IgnoreWhiteSpace = true;
Token current = null;
do
{
current = tokenizer.Next();
if (current.Kind == TokenKind.EOL)
{
}
IToken token = new TokenImpl()
{
Content = current.Value,
Location = new SourceLocation()
{
Column = current.Column, Line = current.Line
},
Type = (TokenType)(int)current.Kind,
};
tokens.Add(token);
} while (current.Kind != TokenKind.EOF);
return(tokens);
}
