public void Add(TokenDefinition <T> tokenDefinition)
{
if (tokenDefinition == null)
{
throw new ArgumentNullException(nameof(tokenDefinition));
}
if (tokenDefinition.TokenType.Equals(invalidToken) || tokenDefinition.TokenType.Equals(eolToken) || tokenDefinition.TokenType.Equals(eofToken))
{
throw new ArgumentException($"{nameof(Tokenizer<T>)} can't register a reserved token (invalid, eol, eof): {typeof(T).Name}.{tokenDefinition.TokenType}", nameof(tokenDefinition));
}
if (tokenDefinitions.Contains(tokenDefinition))
{
throw new ArgumentException($"{nameof(Tokenizer<T>)} can't register duplicate token definitions: {typeof(T).Name}.{tokenDefinition.TokenType} '{tokenDefinition.Regex}'", nameof(tokenDefinition));
}
tokenDefinitions.Add(tokenDefinition);
OnDefinitionAdded?.Invoke(this, tokenDefinition);
}