public IEnumerable <Token> Tokenize(TextReader textReader)
{
ArgumentUtility.IsNotNull(nameof(textReader), textReader);
int readChars;
while ((readChars = textReader.Read(_buffer, 0, ChunkSize)) > 0)
{
foreach (var token in TokenizeBuffer(readChars))
{
yield return(token);
}
}
var lastToken = _tokenizerStateMachine.Finish();
if (lastToken != null)
{
yield return(lastToken.Value);
}
yield return(Token.DelimiterToken);
}
