public void StringTokenizationHandler_EnsureFailure_Test() { RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF); Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions); tokenizer.Handlers.Add(new StringLiteralTokenizationHandler()); tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>()); Token[] tokens = tokenizer.Tokenize("@\"asd\"\"asd\" \"asd\\\"asd\" asd"); Assert.AreEqual(true, tokens[tokens.Length - 1].PreviousNWS.Definition == TokenKind.Unknown); }
public void NumericTokenizationHandler_EnsureFailure_Test() { RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF); Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions); tokenizer.Handlers.Add(new CSharpNumericLiteralTokenizationHandler()); tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>()); Token[] tokens = tokenizer.Tokenize("10 10.0 10.0d 10.0D 10.0f 10.0F 10u 10U 10ul 10UL 10lu 10LU 10l 10L 10e5 10e-5 asd"); Assert.AreEqual(true, tokens[tokens.Length - 1].PreviousNWS.Definition == TokenKind.Unknown); }
public void CharTokenizationHandler_EnsureFailure_Test() { RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF); Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions); CharacterLiteralTokenizationHandler handler = new CharacterLiteralTokenizationHandler(); tokenizer.Handlers.Add(handler); tokenizer.Handlers.Add(handler); tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>()); Token[] tokens = tokenizer.Tokenize("\'r\' \'\\u0066\' \'\\\'\' \'\\x000a\' asd"); Assert.AreEqual(true, tokens[tokens.Length - 1].PreviousNWS.Definition == TokenKind.Unknown); }
public void NumericTokenizationHandler_EnsureSuccess_Test() { RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF); Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions); tokenizer.Handlers.Add(new CSharpNumericLiteralTokenizationHandler()); tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>()); Token[] tokens = tokenizer.Tokenize("10 10.0 10.0d 10.0D 10.0f 10.0F 10u 10U 10ul 10UL 10lu 10LU 10l 10L 10e5 10e-5"); for (Token i = tokens[0]; !i.Definition.IsEndOfFile; i = i.NextNWS) { Assert.AreEqual(true, i.Definition.IsRealLiteral || i.Definition.IsIntegerLiteral); } }
public void StringTokenizationHandler_EnsureSuccess_Test() { RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF); Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions); tokenizer.Handlers.Add(new StringLiteralTokenizationHandler()); tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>()); Token[] tokens = tokenizer.Tokenize("@\"asd\"\"asd\" \"asd\\\"asd\""); for (Token i = tokens[0]; !i.Definition.IsEndOfFile; i = i.NextNWS) { Assert.AreEqual(true, i.Definition.IsStringLiteral); } }
public void CharTokenizationHandler_EnsureSuccess_Test() { RequiredTokenDefinitions <TokenKind> definitions = new RequiredTokenDefinitions <TokenKind>(TokenKind.Whitespace, TokenKind.LineBreak, TokenKind.Unknown, TokenKind.EOF); Tokenizer <Token, TokenKind> tokenizer = new Tokenizer <Token, TokenKind>(definitions); CharacterLiteralTokenizationHandler handler = new CharacterLiteralTokenizationHandler(); tokenizer.Handlers.Add(handler); tokenizer.Handlers.Add(handler); tokenizer.Handlers.Add(new WhitespaceTokenizationHandler <Token, TokenKind>()); Token[] tokens = tokenizer.Tokenize("\'r\' \'\\u0066\' \'\\\'\' \'\\x000a\' "); for (Token i = tokens[0]; !i.Definition.IsEndOfFile; i = i.NextNWS) { Assert.AreEqual(true, i.Definition.IsCharLiteral); } }