public void TestNumber12()
{
var lex = new PythonTokenizer("4_5.3_4_5E-3_5J ".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.Number, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(15u, tok1.End);
}
public void TestString2()
{
var lex = new PythonTokenizer("\"\" ".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.String, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(2u, tok1.End);
}
public void TestNumberBinaryNumber4()
{
var lex = new PythonTokenizer("0B_1_1 ".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.Number, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(6u, tok1.End);
}
public void TestReservedKeywords_return()
{
var lex = new PythonTokenizer("return ".ToCharArray(), false, 8);
lex.Advance();
Assert.Equal(Token.TokenKind.PyReturn, lex.CurSymbol.Kind);
Assert.Equal(0u, lex.CurSymbol.Start);
Assert.Equal(6u, lex.CurSymbol.End);
}
public void TestNumberZero_Dot()
{
var lex = new PythonTokenizer("0. ".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.Number, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(2u, tok1.End);
}
public void TestReservedKeywords_async()
{
var lex = new PythonTokenizer("async ".ToCharArray(), false, 8);
lex.Advance();
Assert.Equal(Token.TokenKind.PyAsync, lex.CurSymbol.Kind);
Assert.Equal(0u, lex.CurSymbol.Start);
Assert.Equal(5u, lex.CurSymbol.End);
}
public void TestNumberExponent4()
{
var lex = new PythonTokenizer("0E+3 ".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.Number, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(4u, tok1.End);
}
public void TestOperatorOrDelimiterComma()
{
var lex = new PythonTokenizer(",".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.PyComma, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(1u, tok1.End);
}
public void TestOperatorOrDelimiterElipsis()
{
var lex = new PythonTokenizer("...".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.PyElipsis, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(3u, tok1.End);
}
public void TestOperatorOrDelimiterFloorDiv()
{
var lex = new PythonTokenizer("// ".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.PyFloorDiv, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(2u, tok1.End);
}
public void TestOperatorOrDelimiterShiftRightAssign()
{
var lex = new PythonTokenizer(">>=".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.PyShiftRightAssign, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(3u, tok1.End);
}
public void TestNumberHexNumber2()
{
var lex = new PythonTokenizer("0X_f_f_a_4_7_7 ".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.Number, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(14u, tok1.End);
}
public void TestReservedKeywords_with()
{
var lex = new PythonTokenizer("with ".ToCharArray(), false, 8);
lex.Advance();
Assert.Equal(Token.TokenKind.PyWith, lex.CurSymbol.Kind);
Assert.Equal(0u, lex.CurSymbol.Start);
Assert.Equal(4u, lex.CurSymbol.End);
}
public void TestReservedKeywords_nonlocal()
{
var lex = new PythonTokenizer("nonlocal ".ToCharArray(), false, 8);
lex.Advance();
Assert.Equal(Token.TokenKind.PyNonlocal, lex.CurSymbol.Kind);
Assert.Equal(0u, lex.CurSymbol.Start);
Assert.Equal(8u, lex.CurSymbol.End);
}
public void TestReservedKeywords_LiteralNameWithDigit()
{
var lex = new PythonTokenizer("N3ame_ ".ToCharArray(), false, 8);
lex.Advance();
Assert.Equal(Token.TokenKind.Name, lex.CurSymbol.Kind);
Assert.Equal(0u, lex.CurSymbol.Start);
Assert.Equal(6u, lex.CurSymbol.End);
}
public void TestReservedKeywords_finally()
{
var lex = new PythonTokenizer("finally ".ToCharArray(), false, 8);
lex.Advance();
Assert.Equal(Token.TokenKind.PyFinally, lex.CurSymbol.Kind);
Assert.Equal(0u, lex.CurSymbol.Start);
Assert.Equal(7u, lex.CurSymbol.End);
}
public void TestTypeComment()
{
var lex = new PythonTokenizer("# type: int\r\n\0 ".ToCharArray(), false, 8);
lex.Advance();
Assert.Equal(Token.TokenKind.TypeComment, lex.CurSymbol.Kind);
Assert.Equal(0u, lex.CurSymbol.Start);
Assert.Equal(11u, lex.CurSymbol.End);
Assert.Equal("# type: int", lex.CurSymbol.Text);
}
public void TestString4()
{
var lex = new PythonTokenizer("\"\"\"\"\"\" ".ToCharArray(), false, 8);
lex.Advance();
Assert.Equal(Token.TokenKind.String, lex.CurSymbol.Kind);
Assert.Equal(0u, lex.CurSymbol.Start);
Assert.Equal(6u, lex.CurSymbol.End);
Assert.Equal("\"\"\"\"\"\"", lex.CurSymbol.Text);
}
public void TestLevelCurlyMissingStart()
{
var lex = new PythonTokenizer("}".ToCharArray(), false, 8);
try
{
var tok1 = lex.GetSymbol();
Assert.True(false);
}
catch
{
Assert.True(true);
}
}
public void TestLevelCurlys()
{
var lex = new PythonTokenizer("{}".ToCharArray(), false, 8);
var tok1 = lex.GetSymbol();
Assert.Equal(Token.TokenKind.PyLeftCurly, tok1.Kind);
Assert.Equal(0u, tok1.Start);
Assert.Equal(1u, tok1.End);
var tok2 = lex.GetSymbol();
Assert.Equal(1u, tok2.Start);
Assert.Equal(2u, tok2.End);
Assert.Equal(Token.TokenKind.PyRightCurly, tok2.Kind);
}
