public void FromSystemConsoleImportThirdTokenValueIsDotCharacter() { FromSystemConsoleImportSecondTokenValueIsSystem(); IPyCompiler.Token token = tokenizer.GetNextToken(); Assert.AreEqual(IPyCompiler.TokenKind.Dot, token.Kind); }
public void TokenAfterFromIsEof() { FirstTokenIsFrom(); IPyCompiler.Token token = tokenizer.GetNextToken(); Assert.AreEqual(IPyCompiler.TokenKind.EndOfFile, token.Kind); }
public void FromSystemImportSecondTokenValueIsSystem() { string text = "from System"; IPyCompiler.Token token = GetSecondToken(text); Assert.AreEqual("System", token.Value as String); }
public void FromSystemImportThirdTokenIsImportToken() { string text = "from System import"; IPyCompiler.Token token = GetThirdToken(text); Assert.AreEqual(IPyCompiler.TokenKind.KeywordImport, token.Kind); }
public void FromSystemImportIdentifierFourthTokenIsIndentifierToken() { string text = "from System import abc"; IPyCompiler.Token token = GetFourthToken(text); Assert.AreEqual("abc", token.Value as String); }
public void FirstTokenIsFrom() { string text = "from"; tokenizer = CreateTokenizer(text); IPyCompiler.Token token = tokenizer.GetNextToken(); Assert.AreEqual(IPyCompiler.TokenKind.KeywordFrom, token.Kind); }
public void FromSystemImportSecondTokenIsModule() { string text = "from System"; IPyCompiler.Token token = GetSecondToken(text); Assert.AreEqual(IPyCompiler.TokenKind.Name, token.Kind); }
IPyCompiler.Token GetToken(string text, int tokenNumber) { tokenizer = CreateTokenizer(text); IPyCompiler.Token token = null; for (int i = 0; i < tokenNumber; ++i) { token = tokenizer.GetNextToken(); } return(token); }
private void ReportSyntaxError(Token t) { ReportSyntaxError(t, ErrorCodes.SyntaxError); }
public bool IsFromToken(Token token) { return token.Kind == IronPython.Compiler.TokenKind.KeywordFrom; }
private void ReportSyntaxError(Token t, int errorCode, bool allowIncomplete) { Location start = GetStart(); if (t.Kind == TokenKind.NewLine || t.Kind == TokenKind.Dedent) { if (tokenizer.IsEndOfFile) { t = EatEndOfInput(); } } if (allowIncomplete && t.Kind == TokenKind.EndOfFile) { errorCode |= ErrorCodes.IncompleteStatement; } ReportSyntaxError(start, tokenizer.EndLocation, String.Format("unexpected token {0}", t.Image), errorCode); }
private void ReportSyntaxError(Token t, int errorCode) { ReportSyntaxError(t, errorCode, true); }
private IClassificationType GetClassificationType(Token token) { // Translate the token kind into a classfication type switch (token.Kind) { case TokenKind.Comment: return classificationRegistryService.GetClassificationType(PyClassificationTypes.Comment); case TokenKind.Dot: case TokenKind.LeftParenthesis: case TokenKind.RightParenthesis: case TokenKind.LeftBracket: case TokenKind.RightBracket: case TokenKind.LeftBrace: case TokenKind.RightBrace: case TokenKind.Comma: case TokenKind.Colon: case TokenKind.BackQuote: case TokenKind.Semicolon: case TokenKind.Assign: case TokenKind.Twiddle: case TokenKind.LessThanGreaterThan: return classificationRegistryService.GetClassificationType(PyClassificationTypes.Delimiter); case TokenKind.Add: case TokenKind.AddEqual: case TokenKind.Subtract: case TokenKind.SubtractEqual: case TokenKind.Power: case TokenKind.PowerEqual: case TokenKind.Multiply: case TokenKind.MultiplyEqual: case TokenKind.FloorDivide: case TokenKind.FloorDivideEqual: case TokenKind.Divide: case TokenKind.DivEqual: case TokenKind.Mod: case TokenKind.ModEqual: case TokenKind.LeftShift: case TokenKind.LeftShiftEqual: case TokenKind.RightShift: case TokenKind.RightShiftEqual: case TokenKind.BitwiseAnd: case TokenKind.BitwiseAndEqual: case TokenKind.BitwiseOr: case TokenKind.BitwiseOrEqual: case TokenKind.Xor: case TokenKind.XorEqual: case TokenKind.LessThan: case TokenKind.GreaterThan: case TokenKind.LessThanOrEqual: case TokenKind.GreaterThanOrEqual: case TokenKind.Equal: case TokenKind.NotEqual: return classificationRegistryService.GetClassificationType(PyClassificationTypes.Operator); case TokenKind.KeywordAnd: case TokenKind.KeywordAssert: case TokenKind.KeywordBreak: case TokenKind.KeywordClass: case TokenKind.KeywordContinue: case TokenKind.KeywordDef: case TokenKind.KeywordDel: case TokenKind.KeywordElseIf: case TokenKind.KeywordElse: case TokenKind.KeywordExcept: case TokenKind.KeywordExec: case TokenKind.KeywordFinally: case TokenKind.KeywordFor: case TokenKind.KeywordFrom: case TokenKind.KeywordGlobal: case TokenKind.KeywordIf: case TokenKind.KeywordImport: case TokenKind.KeywordIn: case TokenKind.KeywordIs: case TokenKind.KeywordLambda: case TokenKind.KeywordNot: case TokenKind.KeywordOr: case TokenKind.KeywordPass: case TokenKind.KeywordPrint: case TokenKind.KeywordRaise: case TokenKind.KeywordReturn: case TokenKind.KeywordTry: case TokenKind.KeywordWhile: case TokenKind.KeywordYield: return classificationRegistryService.GetClassificationType(PyClassificationTypes.Keyword); case TokenKind.Name: return classificationRegistryService.GetClassificationType(PyClassificationTypes.Identifier); case TokenKind.Constant: ConstantValueToken ctoken = (ConstantValueToken)token; if (ctoken.Constant is string) { return classificationRegistryService.GetClassificationType(PyClassificationTypes.String); } else { return classificationRegistryService.GetClassificationType(PyClassificationTypes.Number); } default: return classificationRegistryService.GetClassificationType(PyClassificationTypes.Unknown); } }
// private bool isExprStmtSep(Token t) { // return isEndOfStmt(t) || getAssignOp(t) != null || t == TokenKind.AssignToken; // } // // private bool isEndOfStmt(Token t) { // return t.kind == TokenKind.NEWLINE || t.kind == TokenKind.Semicolon; // } private static BinaryOperator GetAssignOp(Token t) { switch (t.Kind) { case TokenKind.AddEqual: return PythonOperator.Add; case TokenKind.SubtractEqual: return PythonOperator.Subtract; case TokenKind.MultiplyEqual: return PythonOperator.Multiply; case TokenKind.DivEqual: return PythonOperator.Divide; case TokenKind.ModEqual: return PythonOperator.Mod; case TokenKind.BitwiseAndEqual: return PythonOperator.BitwiseAnd; case TokenKind.BitwiseOrEqual: return PythonOperator.BitwiseOr; case TokenKind.XorEqual: return PythonOperator.Xor; case TokenKind.LeftShiftEqual: return PythonOperator.LeftShift; case TokenKind.RightShiftEqual: return PythonOperator.RightShift; case TokenKind.PowerEqual: return PythonOperator.Power; case TokenKind.FloorDivideEqual: return PythonOperator.FloorDivide; default: return null; } }
private static string GetErrorMessage(Token t, int errorCode) { string msg; if ((errorCode & ~ErrorCodes.IncompleteMask) == ErrorCodes.IndentationError) { msg = Resources.ExpectedIndentation; } else if (t.Kind != TokenKind.EndOfFile) { msg = Resources.UnexpectedToken; } else { msg = "unexpected EOF while parsing"; } return msg; }
private Token PeekToken() { if (peekedToken != null) return peekedToken; savedStart = tokenizer.StartLocation; savedEnd = tokenizer.EndLocation; savedExternal = tokenizer.ExternalLineLocation; Token p = NextToken(); peekedToken = p; return p; }
private void ReportSyntaxError(Token t, IndexSpan span, int errorCode, bool allowIncomplete) { var start = span.Start; var end = span.End; if (allowIncomplete && (t.Kind == TokenKind.EndOfFile || (_tokenizer.IsEndOfFile && (t.Kind == TokenKind.Dedent || t.Kind == TokenKind.NLToken)))) { errorCode |= ErrorCodes.IncompleteStatement; } string msg = String.Format(System.Globalization.CultureInfo.InvariantCulture, GetErrorMessage(t, errorCode), t.Image); ReportSyntaxError(start, end, msg, errorCode); }
public TokenWithSpan(Token token, SourceSpan span) { _token = token; _span = span; }
private void DumpToken(Token token) { Console.WriteLine("{0} `{1}`", token.Kind, token.Image.Replace("\r", "\\r").Replace("\n", "\\n").Replace("\t", "\\t")); }
public bool IsImportToken(Token token) { return token.Kind == IronPython.Compiler.TokenKind.KeywordImport; }
public Token GetNextToken() { currentToken = tokenizer.GetNextToken(); return currentToken; }
public bool IsDotToken(Token token) { return token.Kind == IronPython.Compiler.TokenKind.Dot; }
public bool IsNameToken(Token token) { return token.Kind == IronPython.Compiler.TokenKind.Name; }
private static bool NeverTestToken(Token t) { switch (t.Kind) { case TokenKind.AddEqual: case TokenKind.SubtractEqual: case TokenKind.MultiplyEqual: case TokenKind.DivideEqual: case TokenKind.ModEqual: case TokenKind.BitwiseAndEqual: case TokenKind.BitwiseOrEqual: case TokenKind.ExclusiveOrEqual: case TokenKind.LeftShiftEqual: case TokenKind.RightShiftEqual: case TokenKind.PowerEqual: case TokenKind.FloorDivideEqual: case TokenKind.Indent: case TokenKind.Dedent: case TokenKind.NewLine: case TokenKind.EndOfFile: case TokenKind.Semicolon: case TokenKind.Assign: case TokenKind.RightBrace: case TokenKind.RightBracket: case TokenKind.RightParenthesis: case TokenKind.Comma: case TokenKind.KeywordFor: case TokenKind.KeywordIn: case TokenKind.KeywordIf: return true; default: return false; } }
public TokenWithSpan(Token token, IndexSpan span) { _token = token; _span = span; }
private bool PeekToken(Token check) { return PeekToken() == check; }
private void ReportSyntaxError(Token t, SourceSpan span, int errorCode, bool allowIncomplete) { SourceLocation start = span.Start; SourceLocation end = span.End; if (t.Kind == TokenKind.NewLine || t.Kind == TokenKind.Dedent) { if (_tokenizer.IsEndOfFile) { t = EatEndOfInput(); end = _token.Span.End; } } if (allowIncomplete && t.Kind == TokenKind.EndOfFile) { errorCode |= ErrorCodes.IncompleteStatement; } string msg; switch (errorCode) { case ErrorCodes.IndentationError: msg = Resources.ExpectedIndentation; break; default: msg = Resources.UnexpectedToken; break; } ReportSyntaxError(start, end, String.Format(System.Globalization.CultureInfo.InvariantCulture, msg, t.Image), errorCode); }
private PythonOperator GetAssignOperator(Token t) { switch (t.Kind) { case TokenKind.AddEqual: return PythonOperator.Add; case TokenKind.SubtractEqual: return PythonOperator.Subtract; case TokenKind.MultiplyEqual: return PythonOperator.Multiply; case TokenKind.DivideEqual: return TrueDivision ? PythonOperator.TrueDivide : PythonOperator.Divide; case TokenKind.ModEqual: return PythonOperator.Mod; case TokenKind.BitwiseAndEqual: return PythonOperator.BitwiseAnd; case TokenKind.BitwiseOrEqual: return PythonOperator.BitwiseOr; case TokenKind.ExclusiveOrEqual: return PythonOperator.Xor; case TokenKind.LeftShiftEqual: return PythonOperator.LeftShift; case TokenKind.RightShiftEqual: return PythonOperator.RightShift; case TokenKind.PowerEqual: return PythonOperator.Power; case TokenKind.FloorDivideEqual: return PythonOperator.FloorDivide; default: return PythonOperator.None; } }
private Token NextToken() { if (peekedToken != null) { Token ret = peekedToken; peekedToken = null; return ret; } return tokenizer.Next(); }