public AbstractTokenDefinitionTests() { ReservedProvider = new ReservedRegexProvider(); OperatorProvider = new OperatorRegexProvider(); SyntaxProvider = new SyntaxOperatorRegexProvider(); IntegerRegex = RegexWrapper.DefaultWrap(@"\d+\b"); StringRegex = RegexWrapper.DefaultWrap("\"{1}(?:(?:[^\"\\\\]|(?:\\\\[tn\"'\\\\]))+)\"{1}"); IdentifierRegex = RegexWrapper.DefaultWrap(@"(?:_+[a-zA-Z\d]\w*)|(?:[a-zA-Z]\w*)"); }
public Tokenizer(ReservedRegexProvider reservedProvider, OperatorRegexProvider operatorProvider, SyntaxOperatorRegexProvider syntaxProvider) { var reservedPattern = reservedProvider.GetPattern(); RegexTokenDefinition = new TokenDefinition(TokenType.Reserved, RegexWrapper.DefaultWrap(reservedPattern)); var operatorPattern = operatorProvider.GetPattern(); var syntaxPattern = syntaxProvider.GetPattern(); _tokenDefinitions = new List <TokenDefinition> { new TokenDefinition(TokenType.String, RegexWrapper.DefaultWrap(StringPattern)), new TokenDefinition(TokenType.Identifier, RegexWrapper.DefaultWrap(IdentifierPattern)), new TokenDefinition(TokenType.Integer, RegexWrapper.DefaultWrap(IntegerPattern)), new TokenDefinition(TokenType.ArithmeticAndLogicOperator, RegexWrapper.DefaultWrap(operatorPattern)), new TokenDefinition(TokenType.SyntaxOperator, RegexWrapper.DefaultWrap(syntaxPattern)), }; }