public AbstractTokenDefinitionTests()
{
ReservedProvider = new ReservedRegexProvider();
OperatorProvider = new OperatorRegexProvider();
SyntaxProvider = new SyntaxOperatorRegexProvider();
IntegerRegex = RegexWrapper.DefaultWrap(@"\d+\b");
StringRegex = RegexWrapper.DefaultWrap("\"{1}(?:(?:[^\"\\\\]|(?:\\\\[tn\"'\\\\]))+)\"{1}");
IdentifierRegex = RegexWrapper.DefaultWrap(@"(?:_+[a-zA-Z\d]\w*)|(?:[a-zA-Z]\w*)");
}
public void TokenDefinition_IsInvalid1()
{
var input = "fake_int a=32;";
var reservedRegex = RegexWrapper.DefaultWrap(ReservedProvider.GetPattern());
var tokenDefinition = new TokenDefinition(TokenType.Reserved, reservedRegex);
var match = tokenDefinition.Match(input);
Assert.IsFalse(match.IsMatch);
Assert.IsNull(match.Value);
Assert.IsNull(match.RemainingText);
Assert.AreEqual(TokenType.Invalid, match.TokenType);
}
public void TokenDefinition_IsReserved_Integer()
{
var input = "int a=32;";
var reservedRegex = RegexWrapper.DefaultWrap(ReservedProvider.GetPattern());
var tokenDefinition = new TokenDefinition(TokenType.Reserved, reservedRegex);
var match = tokenDefinition.Match(input);
Assert.IsTrue(match.IsMatch);
Assert.AreEqual("int", match.Value);
Assert.AreEqual(" a=32;", match.RemainingText);
Assert.AreEqual(TokenType.Reserved, match.TokenType);
}
public void TokenDefinition_IsReserved_ReturnWithNewline()
{
var input = "return\nasdf";
var reservedRegex = RegexWrapper.DefaultWrap(ReservedProvider.GetPattern());
var tokenDefinition = new TokenDefinition(TokenType.Reserved, reservedRegex);
var match = tokenDefinition.Match(input);
Assert.IsTrue(match.IsMatch);
Assert.AreEqual("return", match.Value);
Assert.AreEqual("\nasdf", match.RemainingText);
Assert.AreEqual(TokenType.Reserved, match.TokenType);
}
public void TokenDefinition_IsOperator2()
{
string[] input =
{
"{ 3 }"
};
var syntaxRegex = RegexWrapper.DefaultWrap(SyntaxProvider.GetPattern());
var tokenDefinition = new TokenDefinition(TokenType.SyntaxOperator, syntaxRegex);
var tokensGenerated = _tokenizer.Tokenize(input).ToList();
var operatorTokens = tokensGenerated.Where(t => t.TokenType == TokenType.SyntaxOperator).ToList();
Assert.IsTrue(operatorTokens.Any());
Assert.AreEqual(2, operatorTokens.Count());
Assert.IsNotNull(operatorTokens.FirstOrDefault(t => t.Value == "{"));
Assert.IsNotNull(operatorTokens.FirstOrDefault(t => t.Value == "}"));
}
public void TokenDefinition_IsOperator_Arithmetic()
{
string[] input =
{
"2 * 5",
"=="
};
var operatorRegex = RegexWrapper.DefaultWrap(OperatorProvider.GetPattern());
var tokenDefinition = new TokenDefinition(TokenType.ArithmeticAndLogicOperator, operatorRegex);
var tokensGenerated = _tokenizer.Tokenize(input).ToList();
var operatorTokens = tokensGenerated.Where(t => t.TokenType == TokenType.ArithmeticAndLogicOperator).ToList();
Assert.IsTrue(operatorTokens.Any());
Assert.AreEqual(2, operatorTokens.Count());
Assert.IsNotNull(operatorTokens.FirstOrDefault(t => t.Value == "*"));
Assert.IsNotNull(operatorTokens.FirstOrDefault(t => t.Value == "=="));
}
