public Tokenizer(SimpleStringReader reader, ImmutableArray <TokenType> delimiters)
{
Assumes.NotNull(reader);
_reader = reader;
_delimiters = delimiters;
}
private Token?GetTokenFrom(SimpleStringReader reader)
{
if (reader.CanRead)
{
return(GetToken(reader.Read()));
}
return(null);
}
private Token?PeekToken()
{
SimpleStringReader reader = _reader.Clone();
return(GetTokenFrom(reader));
}