public Tokenizer(SimpleStringReader reader, ImmutableArray <TokenType> delimiters) { Assumes.NotNull(reader); _reader = reader; _delimiters = delimiters; }
private Token?GetTokenFrom(SimpleStringReader reader) { if (reader.CanRead) { return(GetToken(reader.Read())); } return(null); }
private Token?PeekToken() { SimpleStringReader reader = _reader.Clone(); return(GetTokenFrom(reader)); }