public static IEnumerable <Token> Tokenize(IEnumerable <char> text) { var stateMachine = new StateMachine(); foreach (var symbol in text) { foreach (Token token in stateMachine.Accept(symbol)) { yield return(token); } } foreach (Token token in stateMachine.AcceptEndOfStream()) { yield return(token); } }