private IEnumerable <Token> TokenizeBuffer(int readChars)
{
for (var index = 0; index < readChars; index++)
{
var character = _buffer[index];
var result = _tokenizerStateMachine.AcceptNextCharacter(character);
if (result.HasValue)
{
yield return(result.Value);
if (_tokenizerStateMachine.ShouldEmitNewRecordToken())
{
yield return(Token.DelimiterToken);
}
}
}
}
