internal virtual IEnumerable <Token> Tokenize(CharReader reader)
{
Compile();
if (dfa.Start == null)
{
yield break;
}
PositionCounter start = new PositionCounter();
PositionCounter end = null;
int[] tokenTypeIDs = null;
DFA.State current = dfa.Start;
while (!reader.IsEnd)
{
char c = reader.Read();
current = current[c];
// reached the end, nowhere else to go
// return the match until the prev final state
// and go back to the next char right after the prev final state
if (current == null)
{
yield return(CreateToken(reader, start, end, tokenTypeIDs));
reader.MoveBack(end.Position);
reader.Release();
start = reader.PositionCounter;
end = null;
tokenTypeIDs = null;
current = dfa.Start;
continue;
}
// remember this position in case we need to come back
if (current.IsFinal)
{
end = reader.PositionCounter;
tokenTypeIDs = current.Values;
}
}
if (end != null)
{
yield return(CreateToken(reader, start, end, tokenTypeIDs));
if (end.Position != reader.Position)
{
reader.MoveBack(end.Position - 1);
foreach (Token token in Tokenize(reader))
{
yield return(token);
}
}
}
else
{
yield return(CreateToken(reader, start, reader.PositionCounter, tokenTypeIDs));
}
}