private void ProcessData()
{
TInput value;
while (LinkedFifoBuffer <TInput> .TryGetValue(ref this.bufferPosition, out value))
{
var newState = this.state;
var newSymbol = this.ProcessStateMachine(ref newState, value);
if (Dfa <TLetter> .IsEndState(newState))
{
if (this.tokenSymbol.HasValue || newState == Dfa <TLetter> .Accept)
{
var position = this.bufferPosition;
this.FlushPendingToken();
if (newState == Dfa <TLetter> .Accept)
{
this.tokenEnd = this.tokenStart = this.bufferPosition = position; // restore position after EOF letter
this.ProcessEof();
}
else
{
continue;
}
}
else
{
this.HandleLexicalError(false);
}
}
this.state = newState;
if (newSymbol.HasValue)
{
this.tokenEnd = this.bufferPosition;
this.tokenSymbol = newSymbol.Value;
}
}
}
