public void StringTokenizationHandler_EnsureFailure_Test()
{
RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF);
Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions);
tokenizer.Handlers.Add(new StringLiteralTokenizationHandler());
tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>());
Token[] tokens = tokenizer.Tokenize("@\"asd\"\"asd\" \"asd\\\"asd\" asd");
Assert.AreEqual(true, tokens[tokens.Length - 1].PreviousNWS.Definition == TokenKind.Unknown);
}
public void NumericTokenizationHandler_EnsureFailure_Test()
{
RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF);
Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions);
tokenizer.Handlers.Add(new CSharpNumericLiteralTokenizationHandler());
tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>());
Token[] tokens = tokenizer.Tokenize("10 10.0 10.0d 10.0D 10.0f 10.0F 10u 10U 10ul 10UL 10lu 10LU 10l 10L 10e5 10e-5 asd");
Assert.AreEqual(true, tokens[tokens.Length - 1].PreviousNWS.Definition == TokenKind.Unknown);
}
public void CharTokenizationHandler_EnsureFailure_Test()
{
RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF);
Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions);
CharacterLiteralTokenizationHandler handler = new CharacterLiteralTokenizationHandler();
tokenizer.Handlers.Add(handler);
tokenizer.Handlers.Add(handler);
tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>());
Token[] tokens = tokenizer.Tokenize("\'r\' \'\\u0066\' \'\\\'\' \'\\x000a\' asd");
Assert.AreEqual(true, tokens[tokens.Length - 1].PreviousNWS.Definition == TokenKind.Unknown);
}
public void NumericTokenizationHandler_EnsureSuccess_Test()
{
RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF);
Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions);
tokenizer.Handlers.Add(new CSharpNumericLiteralTokenizationHandler());
tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>());
Token[] tokens = tokenizer.Tokenize("10 10.0 10.0d 10.0D 10.0f 10.0F 10u 10U 10ul 10UL 10lu 10LU 10l 10L 10e5 10e-5");
for (Token i = tokens[0]; !i.Definition.IsEndOfFile; i = i.NextNWS)
{
Assert.AreEqual(true, i.Definition.IsRealLiteral || i.Definition.IsIntegerLiteral);
}
}
public void StringTokenizationHandler_EnsureSuccess_Test()
{
RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF);
Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions);
tokenizer.Handlers.Add(new StringLiteralTokenizationHandler());
tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>());
Token[] tokens = tokenizer.Tokenize("@\"asd\"\"asd\" \"asd\\\"asd\"");
for (Token i = tokens[0]; !i.Definition.IsEndOfFile; i = i.NextNWS)
{
Assert.AreEqual(true, i.Definition.IsStringLiteral);
}
}
public void CharTokenizationHandler_EnsureSuccess_Test()
{
RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF);
Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions);
CharacterLiteralTokenizationHandler handler = new CharacterLiteralTokenizationHandler();
tokenizer.Handlers.Add(handler);
tokenizer.Handlers.Add(handler);
tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>());
Token[] tokens = tokenizer.Tokenize("\'r\' \'\\u0066\' \'\\\'\' \'\\x000a\' ");
for (Token i = tokens[0]; !i.Definition.IsEndOfFile; i = i.NextNWS)
{
Assert.AreEqual(true, i.Definition.IsCharLiteral);
}
}
