public bool Next()
{
Current = null;
if (reader.GetLength(1) == 0)
{
return(false);
}
do
{
foreach (var tokenizer in tokenizers)
{
var str = tokenizer(reader);
if (str != null)
{
Current = new Token(str, tokenizer, reader.Row, reader.Column);
return(true);
}
if (reader.GetLength(1) == 0) // Indentation strips line breaks
{
return(false);
}
}
}while (tokenizerWhiteSpace(reader) != null);
return(false);
}