IReadOnlyList<IToken> ITokenizer.GetTokens(string text) { List<IToken> tokens = new List<IToken>(); StringTokenizer tokenizer = new StringTokenizer(text); tokenizer.IgnoreWhiteSpace = true; Token current = null; do { current = tokenizer.Next(); if (current.Kind == TokenKind.EOL) { } IToken token = new TokenImpl() { Content = current.Value, Location = new SourceLocation() { Column = current.Column, Line = current.Line }, Type = (TokenType)(int)current.Kind, }; tokens.Add(token); } while (current.Kind != TokenKind.EOF); return tokens; }
IReadOnlyList <IToken> ITokenizer.GetTokens(string text) { List <IToken> tokens = new List <IToken>(); StringTokenizer tokenizer = new StringTokenizer(text); tokenizer.IgnoreWhiteSpace = true; Token current = null; do { current = tokenizer.Next(); if (current.Kind == TokenKind.EOL) { } IToken token = new TokenImpl() { Content = current.Value, Location = new SourceLocation() { Column = current.Column, Line = current.Line }, Type = (TokenType)(int)current.Kind, }; tokens.Add(token); } while (current.Kind != TokenKind.EOF); return(tokens); }