/// <summary> /// Constructor. /// </summary> /// <param name="tokenReader">The token reader to pull the tokens from.</param> public TokenEnumerator(TokenReader tokenReader) { var tokens = new List<Token>(); var token = tokenReader.NextToken(); while (token != Token.None) { tokens.Add(token); token = tokenReader.NextToken(); } _tokens = tokens.ToArray(); }
/// <summary> /// Constructor. /// </summary> /// <param name="tokenReader">The token reader to pull the tokens from.</param> public TokenEnumerator(TokenReader tokenReader) { var tokens = new List <Token>(); var token = tokenReader.NextToken(); while (token != Token.None) { tokens.Add(token); token = tokenReader.NextToken(); } _tokens = tokens.ToArray(); }
static IReadOnlyList<Token> Tokenize(TokenReader tokenReader) { var tokens = new List<Token>(); var token = tokenReader.NextToken(); while (token != Token.None) { tokens.Add(token); token = tokenReader.NextToken(); } return tokens; }