public IEnumerable <Token> Lex(string content)
{
StateContainer state = new StateContainer();
Scanner scanner = new Scanner(content);
Token token;
// Start off in the default state.
state.Push(State("default"));
while ((token = ReadToken(scanner, state)) != null)
{
yield return(token);
}
}
private void ProcessState(Word word, string match, StateContainer state)
{
if (word.doesPopState)
{
state.Pop();
}
if (word.pushState != null)
{
state.Push(word.pushState);
}
if (word.isNewLine)
{
state.NewLine();
}
else
{
state.IncrementChar(match.Length);
}
}