public AbstractTokenDefinitionTests() { ReservedProvider = new ReservedRegexProvider(); OperatorProvider = new OperatorRegexProvider(); SyntaxProvider = new SyntaxOperatorRegexProvider(); IntegerRegex = RegexWrapper.DefaultWrap(@"\d+\b"); StringRegex = RegexWrapper.DefaultWrap("\"{1}(?:(?:[^\"\\\\]|(?:\\\\[tn\"'\\\\]))+)\"{1}"); IdentifierRegex = RegexWrapper.DefaultWrap(@"(?:_+[a-zA-Z\d]\w*)|(?:[a-zA-Z]\w*)"); }
public void TokenDefinition_IsInvalid1() { var input = "fake_int a=32;"; var reservedRegex = RegexWrapper.DefaultWrap(ReservedProvider.GetPattern()); var tokenDefinition = new TokenDefinition(TokenType.Reserved, reservedRegex); var match = tokenDefinition.Match(input); Assert.IsFalse(match.IsMatch); Assert.IsNull(match.Value); Assert.IsNull(match.RemainingText); Assert.AreEqual(TokenType.Invalid, match.TokenType); }
public void TokenDefinition_IsReserved_Integer() { var input = "int a=32;"; var reservedRegex = RegexWrapper.DefaultWrap(ReservedProvider.GetPattern()); var tokenDefinition = new TokenDefinition(TokenType.Reserved, reservedRegex); var match = tokenDefinition.Match(input); Assert.IsTrue(match.IsMatch); Assert.AreEqual("int", match.Value); Assert.AreEqual(" a=32;", match.RemainingText); Assert.AreEqual(TokenType.Reserved, match.TokenType); }
public void TokenDefinition_IsReserved_ReturnWithNewline() { var input = "return\nasdf"; var reservedRegex = RegexWrapper.DefaultWrap(ReservedProvider.GetPattern()); var tokenDefinition = new TokenDefinition(TokenType.Reserved, reservedRegex); var match = tokenDefinition.Match(input); Assert.IsTrue(match.IsMatch); Assert.AreEqual("return", match.Value); Assert.AreEqual("\nasdf", match.RemainingText); Assert.AreEqual(TokenType.Reserved, match.TokenType); }
public void TokenDefinition_IsOperator2() { string[] input = { "{ 3 }" }; var syntaxRegex = RegexWrapper.DefaultWrap(SyntaxProvider.GetPattern()); var tokenDefinition = new TokenDefinition(TokenType.SyntaxOperator, syntaxRegex); var tokensGenerated = _tokenizer.Tokenize(input).ToList(); var operatorTokens = tokensGenerated.Where(t => t.TokenType == TokenType.SyntaxOperator).ToList(); Assert.IsTrue(operatorTokens.Any()); Assert.AreEqual(2, operatorTokens.Count()); Assert.IsNotNull(operatorTokens.FirstOrDefault(t => t.Value == "{")); Assert.IsNotNull(operatorTokens.FirstOrDefault(t => t.Value == "}")); }
public void TokenDefinition_IsOperator_Arithmetic() { string[] input = { "2 * 5", "==" }; var operatorRegex = RegexWrapper.DefaultWrap(OperatorProvider.GetPattern()); var tokenDefinition = new TokenDefinition(TokenType.ArithmeticAndLogicOperator, operatorRegex); var tokensGenerated = _tokenizer.Tokenize(input).ToList(); var operatorTokens = tokensGenerated.Where(t => t.TokenType == TokenType.ArithmeticAndLogicOperator).ToList(); Assert.IsTrue(operatorTokens.Any()); Assert.AreEqual(2, operatorTokens.Count()); Assert.IsNotNull(operatorTokens.FirstOrDefault(t => t.Value == "*")); Assert.IsNotNull(operatorTokens.FirstOrDefault(t => t.Value == "==")); }