public IEnumerable <Token> Parse(string sourceCode)
{
var sequence = _sequenceProvider.CreateSequence(sourceCode);
var state = _lexerStateFactory.CreateInitialState(sequence, _parsers);
while (sequence.HasNext())
{
state = state.Act();
if (state.TokenComplete)
{
yield return(MakeToken(state));
state = state.Trim();
}
}
yield return(MakeToken(_lexerStateFactory.CreateTailToken(sequence, state, _parsers)));
}