public Token(string content, TokenTypeInfo type, int line, int character, int tokenNumber)
{
this.Character = character;
this.Line = line;
this.Content = content;
this.Type = type;
this.TokenNumber = tokenNumber;
}
public Token(string content, TokenTypeInfo type, int line, int character, int tokenNumber)
{
this.Character = character;
this.Line = line;
this.Content = content;
this.Type = type;
this.TokenNumber = tokenNumber;
}
public void AddPattern(int tokenType, string pattern, RegexOptions options, string typeName, Func <string, string> escapeFunction)
{
// "\G" indicates that the pattern must match right at the start of the string.
var modifiedPattern = "\\G" + pattern;
var rx = new Regex(modifiedPattern, options);
var hidden = tokenType < 0; // use token types < 0 to indicate hidden tokens lie WS.
var tokenInfo = new TokenTypeInfo(tokenType, rx, typeName, hidden, escapeFunction);
patterns.Add(tokenInfo);
if (!typeNameRegistry.ContainsKey(tokenType))
{
this.typeNameRegistry.Add(tokenType, new List <string>());
}
typeNameRegistry[tokenType].Add(typeName);
}