public void Tokenize_WithValidReader_ForwardsAllCharactersToStateMachine()
{
var reader = new StringReader("Hello World!");
_sut.Tokenize(reader).ToArray();
A.CallTo(() => _tokenizerStateMachine.AcceptNextCharacter('H')).MustHaveHappened(1, Times.Exactly);
A.CallTo(() => _tokenizerStateMachine.AcceptNextCharacter('!')).MustHaveHappened(1, Times.Exactly);
A.CallTo(() => _tokenizerStateMachine.AcceptNextCharacter(A <char> ._))
.MustHaveHappened("Hello World!".Length, Times.Exactly);
}
private IEnumerable <Token> TokenizeBuffer(int readChars)
{
for (var index = 0; index < readChars; index++)
{
var character = _buffer[index];
var result = _tokenizerStateMachine.AcceptNextCharacter(character);
if (result.HasValue)
{
yield return(result.Value);
if (_tokenizerStateMachine.ShouldEmitNewRecordToken())
{
yield return(Token.DelimiterToken);
}
}
}
}