public IEnumerable <Token> AcceptEndOfStream()
{
if (_state != EState.Start)
{
EState nextState;
EState oldState = _state;
if (DefaultTransitionTable.TryGetValue(_state, out nextState))
{
_state = nextState;
Token token;
if (InterpretAcceptState(out token))
{
yield return(token);
}
else
{
throw new TokenizerException(string.Format("Internal error: DefaultTransitionTable[{0}] gave {1} but InterpretAcceptState returns false.", oldState, _state));
}
}
else
{
throw new TokenizerException(string.Format("Unexpected end of stream in state {0}", _state));
}
}
}
public IEnumerable <Token> Accept(char symbol)
{
EState nextState;
if (TransitionTable.TryGetValue(Tuple.Create(_state, Categorize(symbol)), out nextState))
{
_state = nextState;
if (_state != EState.Start)
{
_buffer.Append(symbol);
Token token;
if (InterpretAcceptState(out token))
{
yield return(token);
}
}
}
else if (DefaultTransitionTable.TryGetValue(_state, out nextState))
{
_state = nextState;
Token token;
if (InterpretAcceptState(out token))
{
yield return(token);
}
//Don't consume the symbol, run the symbol again.
foreach (var t in Accept(symbol))
{
yield return(t);
}
}
else
{
throw new TokenizerException(string.Format("Unexpected symbol '{0}' in state {1}", symbol, _state));
}
}
