public void TestOperatorOrDelimiter_FloorDiv()
{
var lex = new PythonCoreTokenizer("//".ToCharArray());
Assert.Equal(TokenKind.PyFloorDiv, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(2ul, lex.CurSymbol.EndPos);
}
public void TestResrevedKeyword_continue()
{
var lex = new PythonCoreTokenizer("continue ".ToCharArray());
Assert.Equal(TokenKind.PyContinue, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(8ul, lex.CurSymbol.EndPos);
}
public void TestLineContinuation_1()
{
var lex = new PythonCoreTokenizer("\\\r\nFalse".ToCharArray());
Assert.Equal(TokenKind.PyFalse, lex.CurSymbol.Kind);
Assert.Equal(3ul, lex.CurSymbol.StartPos);
Assert.Equal(8ul, lex.CurSymbol.EndPos);
}
public void TestResrevedKeyword_class()
{
var lex = new PythonCoreTokenizer("class".ToCharArray());
Assert.Equal(TokenKind.PyClass, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(5ul, lex.CurSymbol.EndPos);
}
public void TestOperatorOrDelimiter_ShiftLeft()
{
var lex = new PythonCoreTokenizer("<<".ToCharArray());
Assert.Equal(TokenKind.PyShiftLeft, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(2ul, lex.CurSymbol.EndPos);
}
public void TestOperatorOrDelimiter_LeftCurly()
{
var lex = new PythonCoreTokenizer("{".ToCharArray());
Assert.Equal(TokenKind.PyLeftCurly, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(1ul, lex.CurSymbol.EndPos);
}
public void TestOperatorOrDelimiter_BitSemiColon()
{
var lex = new PythonCoreTokenizer(";".ToCharArray());
Assert.Equal(TokenKind.PySemiColon, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(1ul, lex.CurSymbol.EndPos);
}
public void TestOperatorOrDelimiter_BitElipsis()
{
var lex = new PythonCoreTokenizer("...".ToCharArray());
Assert.Equal(TokenKind.PyElipsis, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(3ul, lex.CurSymbol.EndPos);
}
public void TestOperatorOrDelimiter_ShiftRightAssign()
{
var lex = new PythonCoreTokenizer(">>=".ToCharArray());
Assert.Equal(TokenKind.PyShiftRightAssign, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(3ul, lex.CurSymbol.EndPos);
}
public void TestOperatorOrDelimiter_BitXorAssign()
{
var lex = new PythonCoreTokenizer("^=".ToCharArray());
Assert.Equal(TokenKind.PyBitXorAssign, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(2ul, lex.CurSymbol.EndPos);
}
public void TestResrevedKeyword_with()
{
var lex = new PythonCoreTokenizer("with".ToCharArray());
Assert.Equal(TokenKind.PyWith, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(4ul, lex.CurSymbol.EndPos);
}
public void TestEndOfFile()
{
var lex = new PythonCoreTokenizer("".ToCharArray());
Assert.Equal(TokenKind.EndOfFile, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(0ul, lex.CurSymbol.EndPos);
}
public void TestResrevedKeyword_and()
{
var lex = new PythonCoreTokenizer("and".ToCharArray());
Assert.Equal(TokenKind.PyAnd, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(3ul, lex.CurSymbol.EndPos);
}
public void TestResrevedKeyword_return()
{
var lex = new PythonCoreTokenizer("return".ToCharArray());
Assert.Equal(TokenKind.PyReturn, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(6ul, lex.CurSymbol.EndPos);
}
public void TestResrevedKeyword_nonlocal()
{
var lex = new PythonCoreTokenizer("nonlocal".ToCharArray());
Assert.Equal(TokenKind.PyNonLocal, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(8ul, lex.CurSymbol.EndPos);
}
public void TestResrevedKeyword_finally()
{
var lex = new PythonCoreTokenizer("finally".ToCharArray());
Assert.Equal(TokenKind.PyFinally, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(7ul, lex.CurSymbol.EndPos);
}
public void TestOperatorOrDelimiter_RightCurly()
{
var lex = new PythonCoreTokenizer("{}".ToCharArray());
lex.Advance();
Assert.Equal(TokenKind.PyRightCurly, lex.CurSymbol.Kind);
Assert.Equal(1ul, lex.CurSymbol.StartPos);
Assert.Equal(2ul, lex.CurSymbol.EndPos);
}
public void TestLiteralNumber_11()
{
var lex = new PythonCoreTokenizer("0o7114".ToCharArray());
Assert.Equal(TokenKind.Number, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(6ul, lex.CurSymbol.EndPos);
Assert.Equal("0o7114", (lex.CurSymbol as NumberToken).Text);
}
public void TestLiteralName_3()
{
var lex = new PythonCoreTokenizer("_x34a".ToCharArray());
Assert.Equal(TokenKind.Name, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(5ul, lex.CurSymbol.EndPos);
Assert.Equal("_x34a", (lex.CurSymbol as NameToken).Text);
}
public void TestLiteralString_8()
{
var lex = new PythonCoreTokenizer("\"Hello, World!\"".ToCharArray());
Assert.Equal(TokenKind.String, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(15ul, lex.CurSymbol.EndPos);
Assert.Equal("\"Hello, World!\"", (lex.CurSymbol as StringToken).Text);
}
public void TestLiteralNumber_26()
{
var lex = new PythonCoreTokenizer("1_2_3_4.4_5_6".ToCharArray());
Assert.Equal(TokenKind.Number, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(13ul, lex.CurSymbol.EndPos);
Assert.Equal("1_2_3_4.4_5_6", (lex.CurSymbol as NumberToken).Text);
}
public void TestTypeComment()
{
var lex = new PythonCoreTokenizer("# type: (int) -> int\r\n".ToCharArray());
Assert.Equal(TokenKind.TypeComment, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(20ul, lex.CurSymbol.EndPos);
Assert.Equal("# type: (int) -> int", (lex.CurSymbol as TypeCommentToken).Text);
}
public static Node ParseEvalInput(string SourceCode, UInt32 TabSize = 8u)
{
var lex = new PythonCoreTokenizer(SourceCode.ToCharArray());
lex.TabSize = TabSize;
var parser = new PythonCoreParser(lex);
return(parser.ParseEvalInput());
}
public void TestLiteralString_3()
{
var lex = new PythonCoreTokenizer("''''''".ToCharArray());
Assert.Equal(TokenKind.String, lex.CurSymbol.Kind);
Assert.Equal(0ul, lex.CurSymbol.StartPos);
Assert.Equal(6ul, lex.CurSymbol.EndPos);
Assert.Equal("''''''", (lex.CurSymbol as StringToken).Text);
}
