public InputToken(Regex match,
IdPolicy idPolicy,
SemanticTokenType tokenType,
OperationType operationType = OperationType.Operator,
TokenDiscardPolicy discardPolicy = TokenDiscardPolicy.Keep
)
{
m_TokenType = tokenType;
m_DiscardPolicy = discardPolicy;
m_OperationType = operationType;
m_Regex = match;
m_IdPolicy = idPolicy;
}
public void AddToken(string stringToMatch, IdPolicy idPolicy, SemanticTokenType tokenType, OperationType operationType = OperationType.Operator, TokenDiscardPolicy discardPolicy = TokenDiscardPolicy.Keep)
{
m_Tokens.Add(new InputToken(new Regex(stringToMatch, RegexOptions.IgnoreCase), idPolicy, tokenType, operationType, discardPolicy)); // Add tokens in order of precedence
}
public void AddToken(Regex match, IdPolicy IdPolicy, SemanticTokenType tokenType, OperationType operationType = OperationType.Operator, TokenDiscardPolicy discardPolicy = TokenDiscardPolicy.Keep)
{
m_Tokens.Add(new InputToken(match, IdPolicy, tokenType, operationType, discardPolicy)); // Add tokens in order of precedence
}
