public IEnumerable <Token> Parse(string sourceCode) { var sequence = _sequenceProvider.CreateSequence(sourceCode); var state = _lexerStateFactory.CreateInitialState(sequence, _parsers); while (sequence.HasNext()) { state = state.Act(); if (state.TokenComplete) { yield return(MakeToken(state)); state = state.Trim(); } } yield return(MakeToken(_lexerStateFactory.CreateTailToken(sequence, state, _parsers))); }