internal TokenInfo AddToken(RegularExpression definition, Lexer state, int indexInState, string description)
{
int index = m_tokenList.Count;
Token tag = new Token(index, description ?? definition.ToString());
TokenInfo token = new TokenInfo(definition, this, state, tag);
m_tokenList.Add(token);
return(token);
}