public void TestLevelCurlys() { var lex = new PythonTokenizer("{}".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.PyLeftCurly, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(1u, tok1.End); var tok2 = lex.GetSymbol(); Assert.Equal(1u, tok2.Start); Assert.Equal(2u, tok2.End); Assert.Equal(Token.TokenKind.PyRightCurly, tok2.Kind); }
public void TestNumberZero_Dot() { var lex = new PythonTokenizer("0. ".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.Number, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(2u, tok1.End); }
public void TestOperatorOrDelimiterElipsis() { var lex = new PythonTokenizer("...".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.PyElipsis, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(3u, tok1.End); }
public void TestOperatorOrDelimiterComma() { var lex = new PythonTokenizer(",".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.PyComma, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(1u, tok1.End); }
public void TestOperatorOrDelimiterShiftRightAssign() { var lex = new PythonTokenizer(">>=".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.PyShiftRightAssign, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(3u, tok1.End); }
public void TestOperatorOrDelimiterFloorDiv() { var lex = new PythonTokenizer("// ".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.PyFloorDiv, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(2u, tok1.End); }
public void TestString2() { var lex = new PythonTokenizer("\"\" ".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.String, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(2u, tok1.End); }
public void TestNumber12() { var lex = new PythonTokenizer("4_5.3_4_5E-3_5J ".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.Number, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(15u, tok1.End); }
public void TestNumberBinaryNumber4() { var lex = new PythonTokenizer("0B_1_1 ".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.Number, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(6u, tok1.End); }
public void TestNumberHexNumber2() { var lex = new PythonTokenizer("0X_f_f_a_4_7_7 ".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.Number, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(14u, tok1.End); }
public void TestNumberExponent4() { var lex = new PythonTokenizer("0E+3 ".ToCharArray(), false, 8); var tok1 = lex.GetSymbol(); Assert.Equal(Token.TokenKind.Number, tok1.Kind); Assert.Equal(0u, tok1.Start); Assert.Equal(4u, tok1.End); }
public void TestLevelCurlyMissingStart() { var lex = new PythonTokenizer("}".ToCharArray(), false, 8); try { var tok1 = lex.GetSymbol(); Assert.True(false); } catch { Assert.True(true); } }