public void Tokenize_WithValidReader_ForwardsAllCharactersToStateMachine() { var reader = new StringReader("Hello World!"); _sut.Tokenize(reader).ToArray(); A.CallTo(() => _tokenizerStateMachine.AcceptNextCharacter('H')).MustHaveHappened(1, Times.Exactly); A.CallTo(() => _tokenizerStateMachine.AcceptNextCharacter('!')).MustHaveHappened(1, Times.Exactly); A.CallTo(() => _tokenizerStateMachine.AcceptNextCharacter(A <char> ._)) .MustHaveHappened("Hello World!".Length, Times.Exactly); }
private IEnumerable <Token> TokenizeBuffer(int readChars) { for (var index = 0; index < readChars; index++) { var character = _buffer[index]; var result = _tokenizerStateMachine.AcceptNextCharacter(character); if (result.HasValue) { yield return(result.Value); if (_tokenizerStateMachine.ShouldEmitNewRecordToken()) { yield return(Token.DelimiterToken); } } } }