示例#1
0
        public void FromSystemConsoleImportThirdTokenValueIsDotCharacter()
        {
            FromSystemConsoleImportSecondTokenValueIsSystem();
            IPyCompiler.Token token = tokenizer.GetNextToken();

            Assert.AreEqual(IPyCompiler.TokenKind.Dot, token.Kind);
        }
示例#2
0
        public void TokenAfterFromIsEof()
        {
            FirstTokenIsFrom();
            IPyCompiler.Token token = tokenizer.GetNextToken();

            Assert.AreEqual(IPyCompiler.TokenKind.EndOfFile, token.Kind);
        }
示例#3
0
        public void FromSystemImportSecondTokenValueIsSystem()
        {
            string text = "from System";

            IPyCompiler.Token token = GetSecondToken(text);

            Assert.AreEqual("System", token.Value as String);
        }
示例#4
0
        public void FromSystemImportThirdTokenIsImportToken()
        {
            string text = "from System import";

            IPyCompiler.Token token = GetThirdToken(text);

            Assert.AreEqual(IPyCompiler.TokenKind.KeywordImport, token.Kind);
        }
示例#5
0
        public void FromSystemImportIdentifierFourthTokenIsIndentifierToken()
        {
            string text = "from System import abc";

            IPyCompiler.Token token = GetFourthToken(text);

            Assert.AreEqual("abc", token.Value as String);
        }
示例#6
0
        public void FirstTokenIsFrom()
        {
            string text = "from";

            tokenizer = CreateTokenizer(text);
            IPyCompiler.Token token = tokenizer.GetNextToken();
            Assert.AreEqual(IPyCompiler.TokenKind.KeywordFrom, token.Kind);
        }
示例#7
0
        public void FromSystemImportSecondTokenIsModule()
        {
            string text = "from System";

            IPyCompiler.Token token = GetSecondToken(text);

            Assert.AreEqual(IPyCompiler.TokenKind.Name, token.Kind);
        }
示例#8
0
        IPyCompiler.Token GetToken(string text, int tokenNumber)
        {
            tokenizer = CreateTokenizer(text);

            IPyCompiler.Token token = null;
            for (int i = 0; i < tokenNumber; ++i)
            {
                token = tokenizer.GetNextToken();
            }
            return(token);
        }
示例#9
0
 private void ReportSyntaxError(Token t)
 {
     ReportSyntaxError(t, ErrorCodes.SyntaxError);
 }
示例#10
0
		public bool IsFromToken(Token token)
		{
			return token.Kind == IronPython.Compiler.TokenKind.KeywordFrom;
		}
示例#11
0
 private void ReportSyntaxError(Token t, int errorCode, bool allowIncomplete)
 {
     Location start = GetStart();
     if (t.Kind == TokenKind.NewLine || t.Kind == TokenKind.Dedent) {
         if (tokenizer.IsEndOfFile) {
             t = EatEndOfInput();
         }
     }
     if (allowIncomplete && t.Kind == TokenKind.EndOfFile) {
         errorCode |= ErrorCodes.IncompleteStatement;
     }
     ReportSyntaxError(start, tokenizer.EndLocation, String.Format("unexpected token {0}", t.Image), errorCode);
 }
示例#12
0
 private void ReportSyntaxError(Token t, int errorCode)
 {
     ReportSyntaxError(t, errorCode, true);
 }
        private IClassificationType GetClassificationType(Token token)
        {
            // Translate the token kind into a classfication type
            switch (token.Kind)
            {
                case TokenKind.Comment:
                    return classificationRegistryService.GetClassificationType(PyClassificationTypes.Comment);

                case TokenKind.Dot:
                case TokenKind.LeftParenthesis:
                case TokenKind.RightParenthesis:
                case TokenKind.LeftBracket:
                case TokenKind.RightBracket:
                case TokenKind.LeftBrace:
                case TokenKind.RightBrace:
                case TokenKind.Comma:
                case TokenKind.Colon:
                case TokenKind.BackQuote:
                case TokenKind.Semicolon:
                case TokenKind.Assign:
                case TokenKind.Twiddle:
                case TokenKind.LessThanGreaterThan:
                    return classificationRegistryService.GetClassificationType(PyClassificationTypes.Delimiter);

                case TokenKind.Add:
                case TokenKind.AddEqual:
                case TokenKind.Subtract:
                case TokenKind.SubtractEqual:
                case TokenKind.Power:
                case TokenKind.PowerEqual:
                case TokenKind.Multiply:
                case TokenKind.MultiplyEqual:
                case TokenKind.FloorDivide:
                case TokenKind.FloorDivideEqual:
                case TokenKind.Divide:
                case TokenKind.DivEqual:
                case TokenKind.Mod:
                case TokenKind.ModEqual:
                case TokenKind.LeftShift:
                case TokenKind.LeftShiftEqual:
                case TokenKind.RightShift:
                case TokenKind.RightShiftEqual:
                case TokenKind.BitwiseAnd:
                case TokenKind.BitwiseAndEqual:
                case TokenKind.BitwiseOr:
                case TokenKind.BitwiseOrEqual:
                case TokenKind.Xor:
                case TokenKind.XorEqual:
                case TokenKind.LessThan:
                case TokenKind.GreaterThan:
                case TokenKind.LessThanOrEqual:
                case TokenKind.GreaterThanOrEqual:
                case TokenKind.Equal:
                case TokenKind.NotEqual:
                    return classificationRegistryService.GetClassificationType(PyClassificationTypes.Operator);

                case TokenKind.KeywordAnd:
                case TokenKind.KeywordAssert:
                case TokenKind.KeywordBreak:
                case TokenKind.KeywordClass:
                case TokenKind.KeywordContinue:
                case TokenKind.KeywordDef:
                case TokenKind.KeywordDel:
                case TokenKind.KeywordElseIf:
                case TokenKind.KeywordElse:
                case TokenKind.KeywordExcept:
                case TokenKind.KeywordExec:
                case TokenKind.KeywordFinally:
                case TokenKind.KeywordFor:
                case TokenKind.KeywordFrom:
                case TokenKind.KeywordGlobal:
                case TokenKind.KeywordIf:
                case TokenKind.KeywordImport:
                case TokenKind.KeywordIn:
                case TokenKind.KeywordIs:
                case TokenKind.KeywordLambda:
                case TokenKind.KeywordNot:
                case TokenKind.KeywordOr:
                case TokenKind.KeywordPass:
                case TokenKind.KeywordPrint:
                case TokenKind.KeywordRaise:
                case TokenKind.KeywordReturn:
                case TokenKind.KeywordTry:
                case TokenKind.KeywordWhile:
                case TokenKind.KeywordYield:
                    return classificationRegistryService.GetClassificationType(PyClassificationTypes.Keyword);

                case TokenKind.Name:
                    return classificationRegistryService.GetClassificationType(PyClassificationTypes.Identifier);

                case TokenKind.Constant:
                    ConstantValueToken ctoken = (ConstantValueToken)token;
                    if (ctoken.Constant is string)
                    {
                        return classificationRegistryService.GetClassificationType(PyClassificationTypes.String);
                    }
                    else
                    {
                        return classificationRegistryService.GetClassificationType(PyClassificationTypes.Number);
                    }

                default:
                    return classificationRegistryService.GetClassificationType(PyClassificationTypes.Unknown);
            }
        }
示例#14
0
 //        private bool isExprStmtSep(Token t) {
 //            return isEndOfStmt(t) || getAssignOp(t) != null || t == TokenKind.AssignToken;
 //        }
 //
 //        private bool isEndOfStmt(Token t) {
 //            return t.kind == TokenKind.NEWLINE || t.kind == TokenKind.Semicolon;
 //        }
 private static BinaryOperator GetAssignOp(Token t)
 {
     switch (t.Kind) {
         case TokenKind.AddEqual: return PythonOperator.Add;
         case TokenKind.SubtractEqual: return PythonOperator.Subtract;
         case TokenKind.MultiplyEqual: return PythonOperator.Multiply;
         case TokenKind.DivEqual: return PythonOperator.Divide;
         case TokenKind.ModEqual: return PythonOperator.Mod;
         case TokenKind.BitwiseAndEqual: return PythonOperator.BitwiseAnd;
         case TokenKind.BitwiseOrEqual: return PythonOperator.BitwiseOr;
         case TokenKind.XorEqual: return PythonOperator.Xor;
         case TokenKind.LeftShiftEqual: return PythonOperator.LeftShift;
         case TokenKind.RightShiftEqual: return PythonOperator.RightShift;
         case TokenKind.PowerEqual: return PythonOperator.Power;
         case TokenKind.FloorDivideEqual: return PythonOperator.FloorDivide;
         default: return null;
     }
 }
示例#15
0
 private static string GetErrorMessage(Token t, int errorCode) {
     string msg;
     if ((errorCode & ~ErrorCodes.IncompleteMask) == ErrorCodes.IndentationError) {
         msg = Resources.ExpectedIndentation;
     } else if (t.Kind != TokenKind.EndOfFile) {
         msg = Resources.UnexpectedToken;
     } else {
         msg = "unexpected EOF while parsing";
     }
     
     return msg;
 }
示例#16
0
        private Token PeekToken()
        {
            if (peekedToken != null) return peekedToken;

            savedStart = tokenizer.StartLocation;
            savedEnd = tokenizer.EndLocation;
            savedExternal = tokenizer.ExternalLineLocation;

            Token p = NextToken();
            peekedToken = p;

            return p;
        }
示例#17
0
        private void ReportSyntaxError(Token t, IndexSpan span, int errorCode, bool allowIncomplete) {
            var start = span.Start;
            var end = span.End;

            if (allowIncomplete && (t.Kind == TokenKind.EndOfFile || (_tokenizer.IsEndOfFile && (t.Kind == TokenKind.Dedent || t.Kind == TokenKind.NLToken)))) {
                errorCode |= ErrorCodes.IncompleteStatement;
            }

            string msg = String.Format(System.Globalization.CultureInfo.InvariantCulture, GetErrorMessage(t, errorCode), t.Image);

            ReportSyntaxError(start, end, msg, errorCode);
        }
示例#18
0
 public TokenWithSpan(Token token, SourceSpan span) {
     _token = token;
     _span = span;
 }
示例#19
0
 private void DumpToken(Token token) {
     Console.WriteLine("{0} `{1}`", token.Kind, token.Image.Replace("\r", "\\r").Replace("\n", "\\n").Replace("\t", "\\t"));
 }
示例#20
0
		public bool IsImportToken(Token token)
		{
			return token.Kind == IronPython.Compiler.TokenKind.KeywordImport;
		}
示例#21
0
		public Token GetNextToken()
		{
			currentToken = tokenizer.GetNextToken();
			return currentToken;
		}
示例#22
0
		public bool IsDotToken(Token token)
		{
			return token.Kind == IronPython.Compiler.TokenKind.Dot;
		}
示例#23
0
		public bool IsNameToken(Token token)
		{
			return token.Kind == IronPython.Compiler.TokenKind.Name;
		}
示例#24
0
        private static bool NeverTestToken(Token t) {
            switch (t.Kind) {
                case TokenKind.AddEqual:
                case TokenKind.SubtractEqual:
                case TokenKind.MultiplyEqual:
                case TokenKind.DivideEqual:
                case TokenKind.ModEqual:
                case TokenKind.BitwiseAndEqual:
                case TokenKind.BitwiseOrEqual:
                case TokenKind.ExclusiveOrEqual:
                case TokenKind.LeftShiftEqual:
                case TokenKind.RightShiftEqual:
                case TokenKind.PowerEqual:
                case TokenKind.FloorDivideEqual:

                case TokenKind.Indent:
                case TokenKind.Dedent:
                case TokenKind.NewLine:
                case TokenKind.EndOfFile:
                case TokenKind.Semicolon:

                case TokenKind.Assign:
                case TokenKind.RightBrace:
                case TokenKind.RightBracket:
                case TokenKind.RightParenthesis:

                case TokenKind.Comma:

                case TokenKind.KeywordFor:
                case TokenKind.KeywordIn:
                case TokenKind.KeywordIf:
                    return true;

                default: return false;
            }
        }
示例#25
0
 public TokenWithSpan(Token token, IndexSpan span) {
     _token = token;
     _span = span;
 }
示例#26
0
 private bool PeekToken(Token check) {
     return PeekToken() == check;
 }       
示例#27
0
        private void ReportSyntaxError(Token t, SourceSpan span, int errorCode, bool allowIncomplete) {
            SourceLocation start = span.Start;
            SourceLocation end = span.End;
            if (t.Kind == TokenKind.NewLine || t.Kind == TokenKind.Dedent) {
                if (_tokenizer.IsEndOfFile) {
                    t = EatEndOfInput();
                    end = _token.Span.End;
                }
            }

            if (allowIncomplete && t.Kind == TokenKind.EndOfFile) {
                errorCode |= ErrorCodes.IncompleteStatement;
            }

            string msg;
            switch (errorCode) {
                case ErrorCodes.IndentationError: msg = Resources.ExpectedIndentation; break;
                default: msg = Resources.UnexpectedToken; break;
            }

            ReportSyntaxError(start, end, String.Format(System.Globalization.CultureInfo.InvariantCulture,
                msg, t.Image), errorCode);
        }
示例#28
0
 private PythonOperator GetAssignOperator(Token t) {
     switch (t.Kind) {
         case TokenKind.AddEqual: return PythonOperator.Add;
         case TokenKind.SubtractEqual: return PythonOperator.Subtract;
         case TokenKind.MultiplyEqual: return PythonOperator.Multiply;
         case TokenKind.DivideEqual: return TrueDivision ? PythonOperator.TrueDivide : PythonOperator.Divide;
         case TokenKind.ModEqual: return PythonOperator.Mod;
         case TokenKind.BitwiseAndEqual: return PythonOperator.BitwiseAnd;
         case TokenKind.BitwiseOrEqual: return PythonOperator.BitwiseOr;
         case TokenKind.ExclusiveOrEqual: return PythonOperator.Xor;
         case TokenKind.LeftShiftEqual: return PythonOperator.LeftShift;
         case TokenKind.RightShiftEqual: return PythonOperator.RightShift;
         case TokenKind.PowerEqual: return PythonOperator.Power;
         case TokenKind.FloorDivideEqual: return PythonOperator.FloorDivide;
         default: return PythonOperator.None;
     }
 }
示例#29
0
 private Token NextToken()
 {
     if (peekedToken != null) {
         Token ret = peekedToken;
         peekedToken = null;
         return ret;
     }
     return tokenizer.Next();
 }