public AbstractTokenDefinitionTests()
{
ReservedProvider = new ReservedRegexProvider();
OperatorProvider = new OperatorRegexProvider();
SyntaxProvider = new SyntaxOperatorRegexProvider();
IntegerRegex = RegexWrapper.DefaultWrap(@"\d+\b");
StringRegex = RegexWrapper.DefaultWrap("\"{1}(?:(?:[^\"\\\\]|(?:\\\\[tn\"'\\\\]))+)\"{1}");
IdentifierRegex = RegexWrapper.DefaultWrap(@"(?:_+[a-zA-Z\d]\w*)|(?:[a-zA-Z]\w*)");
}
public Tokenizer(ReservedRegexProvider reservedProvider, OperatorRegexProvider operatorProvider, SyntaxOperatorRegexProvider syntaxProvider)
{
var reservedPattern = reservedProvider.GetPattern();
RegexTokenDefinition = new TokenDefinition(TokenType.Reserved, RegexWrapper.DefaultWrap(reservedPattern));
var operatorPattern = operatorProvider.GetPattern();
var syntaxPattern = syntaxProvider.GetPattern();
_tokenDefinitions = new List <TokenDefinition>
{
new TokenDefinition(TokenType.String, RegexWrapper.DefaultWrap(StringPattern)),
new TokenDefinition(TokenType.Identifier, RegexWrapper.DefaultWrap(IdentifierPattern)),
new TokenDefinition(TokenType.Integer, RegexWrapper.DefaultWrap(IntegerPattern)),
new TokenDefinition(TokenType.ArithmeticAndLogicOperator, RegexWrapper.DefaultWrap(operatorPattern)),
new TokenDefinition(TokenType.SyntaxOperator, RegexWrapper.DefaultWrap(syntaxPattern)),
};
}
