public static IEnumerable <Token> Tokenize(IEnumerable <char> text)
{
var stateMachine = new StateMachine();
foreach (var symbol in text)
{
foreach (Token token in stateMachine.Accept(symbol))
{
yield return(token);
}
}
foreach (Token token in stateMachine.AcceptEndOfStream())
{
yield return(token);
}
}