public InputToken(Regex match, SemanticTokenType tokenType, OperationType operationType = OperationType.Operator, TokenDiscardPolicy discardPolicy = TokenDiscardPolicy.Keep)
{
m_TokenType = tokenType;
m_DiscardPolicy = discardPolicy;
m_OperationType = operationType;
m_Regex = match;
}
public Token(Regex match, TokenType tokenType, OperationType operationType = OperationType.Operator, TokenDiscardPolicy discardPolicy = TokenDiscardPolicy.Keep)
{
m_TokenType = tokenType;
m_DiscardPolicy = discardPolicy;
m_OperationType = operationType;
m_Regex = match;
}
public TokenDefinition(TokenType type, OperationType operationType, Regex regex,
TokenDiscardPolicy discardPolicy)
{
m_OperationType = operationType;
m_DiscardPolicy = discardPolicy;
Type = type;
Regex = regex;
}
public void AddToken(string stringToMatch, SemanticTokenType tokenType, OperationType operationType = OperationType.Operator, TokenDiscardPolicy discardPolicy = TokenDiscardPolicy.Keep)
{
m_Tokens.Add(new InputToken(new Regex(stringToMatch, RegexOptions.IgnoreCase), tokenType, operationType, discardPolicy)); // Add tokens in order of precedence
}
public void AddToken(Regex match, SemanticTokenType tokenType, OperationType operationType = OperationType.Operator, TokenDiscardPolicy discardPolicy = TokenDiscardPolicy.Keep)
{
m_Tokens.Add(new InputToken(match, tokenType, operationType, discardPolicy)); // Add tokens in order of precedence
}
