public void TestReservedKeywords_async() { var lex = new PythonTokenizer("async ".ToCharArray(), false, 8); lex.Advance(); Assert.Equal(Token.TokenKind.PyAsync, lex.CurSymbol.Kind); Assert.Equal(0u, lex.CurSymbol.Start); Assert.Equal(5u, lex.CurSymbol.End); }
public void TestReservedKeywords_return() { var lex = new PythonTokenizer("return ".ToCharArray(), false, 8); lex.Advance(); Assert.Equal(Token.TokenKind.PyReturn, lex.CurSymbol.Kind); Assert.Equal(0u, lex.CurSymbol.Start); Assert.Equal(6u, lex.CurSymbol.End); }
public void TestReservedKeywords_with() { var lex = new PythonTokenizer("with ".ToCharArray(), false, 8); lex.Advance(); Assert.Equal(Token.TokenKind.PyWith, lex.CurSymbol.Kind); Assert.Equal(0u, lex.CurSymbol.Start); Assert.Equal(4u, lex.CurSymbol.End); }
public void TestReservedKeywords_nonlocal() { var lex = new PythonTokenizer("nonlocal ".ToCharArray(), false, 8); lex.Advance(); Assert.Equal(Token.TokenKind.PyNonlocal, lex.CurSymbol.Kind); Assert.Equal(0u, lex.CurSymbol.Start); Assert.Equal(8u, lex.CurSymbol.End); }
public void TestReservedKeywords_LiteralNameWithDigit() { var lex = new PythonTokenizer("N3ame_ ".ToCharArray(), false, 8); lex.Advance(); Assert.Equal(Token.TokenKind.Name, lex.CurSymbol.Kind); Assert.Equal(0u, lex.CurSymbol.Start); Assert.Equal(6u, lex.CurSymbol.End); }
public void TestReservedKeywords_finally() { var lex = new PythonTokenizer("finally ".ToCharArray(), false, 8); lex.Advance(); Assert.Equal(Token.TokenKind.PyFinally, lex.CurSymbol.Kind); Assert.Equal(0u, lex.CurSymbol.Start); Assert.Equal(7u, lex.CurSymbol.End); }
public void TestTypeComment() { var lex = new PythonTokenizer("# type: int\r\n\0 ".ToCharArray(), false, 8); lex.Advance(); Assert.Equal(Token.TokenKind.TypeComment, lex.CurSymbol.Kind); Assert.Equal(0u, lex.CurSymbol.Start); Assert.Equal(11u, lex.CurSymbol.End); Assert.Equal("# type: int", lex.CurSymbol.Text); }
public void TestString4() { var lex = new PythonTokenizer("\"\"\"\"\"\" ".ToCharArray(), false, 8); lex.Advance(); Assert.Equal(Token.TokenKind.String, lex.CurSymbol.Kind); Assert.Equal(0u, lex.CurSymbol.Start); Assert.Equal(6u, lex.CurSymbol.End); Assert.Equal("\"\"\"\"\"\"", lex.CurSymbol.Text); }