Beispiel #1
0
        private void ParsePythonCode(
            ITextSnapshot snapshot, TextReader content, Severity indentationSeverity,
            out PythonAst ast, out CollectingErrorSink errorSink, out List<TaskProviderItem> commentTasks
        ) {
            ast = null;
            errorSink = new CollectingErrorSink();
            var tasks = commentTasks = new List<TaskProviderItem>();

            var options = new ParserOptions {
                    ErrorSink = errorSink,
                    IndentationInconsistencySeverity = indentationSeverity,
                BindReferences = true,
            };
            options.ProcessComment += (sender, e) => ProcessComment(tasks, snapshot, e.Span, e.Text);

            using (var parser = Parser.CreateParser(content, Project.LanguageVersion, options)) {
                ast = ParseOneFile(ast, parser);
            }
        }
Beispiel #2
0
        public unsafe void SetValueAsString(DkmEvaluationResult result, string value, int timeout, out string errorText) {
            var pyEvalResult = result.GetDataItem<PyObjectEvaluationResult>();
            if (pyEvalResult == null) {
                Debug.Fail("SetValueAsString called on a DkmEvaluationResult without an associated PyObjectEvaluationResult.");
                throw new NotSupportedException();
            }

            var proxy = pyEvalResult.ValueStore as IWritableDataProxy;
            if (proxy == null) {
                Debug.Fail("SetValueAsString called on a DkmEvaluationResult that does not correspond to an IWritableDataProxy.");
                throw new InvalidOperationException();
            }

            errorText = null;
            var process = result.StackFrame.Process;
            var pyrtInfo = process.GetPythonRuntimeInfo();

            var parserOptions = new ParserOptions { ErrorSink = new StringErrorSink() };
            var parser = Parser.CreateParser(new StringReader(value), pyrtInfo.LanguageVersion, parserOptions);
            var body = (ReturnStatement)parser.ParseTopExpression().Body;
            errorText = parserOptions.ErrorSink.ToString();
            if (!string.IsNullOrEmpty(errorText)) {
                return;
            }

            var expr = body.Expression;
            while (true) {
                var parenExpr = expr as ParenthesisExpression;
                if (parenExpr == null) {
                    break;
                }
                expr = parenExpr.Expression;
            }

            int sign;
            expr = ForceExplicitSign(expr, out sign);

            PyObject newObj = null;

            var constExpr = expr as ConstantExpression;
            if (constExpr != null) {
                if (constExpr.Value == null) {
                    newObj = PyObject.None(process);
                } else if (constExpr.Value is bool) {
                    // In 2.7, 'True' and 'False' are reported as identifiers, not literals, and are handled separately below.
                    newObj = PyBoolObject33.Create(process, (bool)constExpr.Value);
                } else if (constExpr.Value is string) {
                    if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27) {
                        newObj = PyUnicodeObject27.Create(process, (string)constExpr.Value);
                    } else {
                        newObj = PyUnicodeObject33.Create(process, (string)constExpr.Value);
                    }
                } else if (constExpr.Value is AsciiString) {
                    newObj = PyBytesObject.Create(process, (AsciiString)constExpr.Value);
                }
            } else {
                var unaryExpr = expr as UnaryExpression;
                if (unaryExpr != null && sign != 0) {
                    constExpr = unaryExpr.Expression as ConstantExpression;
                    if (constExpr != null) {
                        if (constExpr.Value is BigInteger) {
                            newObj = PyLongObject.Create(process, (BigInteger)constExpr.Value * sign);
                        } else if (constExpr.Value is int) {
                            if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27) {
                                newObj = PyIntObject.Create(process, (int)constExpr.Value * sign);
                            } else {
                                newObj = PyLongObject.Create(process, (int)constExpr.Value * sign);
                            }
                        } else if (constExpr.Value is double) {
                            newObj = PyFloatObject.Create(process, (double)constExpr.Value * sign);
                        } else if (constExpr.Value is Complex) {
                            newObj = PyComplexObject.Create(process, (Complex)constExpr.Value * sign);
                        }
                    }
                } else {
                    var binExpr = expr as BinaryExpression;
                    if (binExpr != null && (binExpr.Operator == PythonOperator.Add || binExpr.Operator == PythonOperator.Subtract)) {
                        int realSign;
                        var realExpr = ForceExplicitSign(binExpr.Left, out realSign) as UnaryExpression;
                        int imagSign;
                        var imagExpr = ForceExplicitSign(binExpr.Right, out imagSign) as UnaryExpression;
                        if (realExpr != null && realSign != 0 && imagExpr != null && imagSign != 0) {
                            var realConst = realExpr.Expression as ConstantExpression;
                            var imagConst = imagExpr.Expression as ConstantExpression;
                            if (realConst != null && imagConst != null) {
                                var realVal = (realConst.Value as int? ?? realConst.Value as double?) as IConvertible;
                                var imagVal = imagConst.Value as Complex?;
                                if (realVal != null && imagVal != null) {
                                    double real = realVal.ToDouble(null) * realSign;
                                    double imag = imagVal.Value.Imaginary * imagSign * (binExpr.Operator == PythonOperator.Add ? 1 : -1);
                                    newObj = PyComplexObject.Create(process, new Complex(real, imag));
                                }
                            }
                        }
                    } else {
                        if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27) {
                            // 'True' and 'False' are not literals in 2.x, but we want to treat them as such.
                            var name = expr as NameExpression;
                            if (name != null) {
                                if (name.Name == "True") {
                                    newObj = PyBoolObject27.Create(process, true);
                                } else if (name.Name == "False") {
                                    newObj = PyBoolObject27.Create(process, false);
                                }
                            }
                        }
                    }
                }
            }

            if (newObj != null) {
                var oldObj = proxy.Read() as PyObject;
                if (oldObj != null) {
                    // We can't free the original value without running some code in the process, and it may be holding heap locks.
                    // So don't decrement refcount now, but instead add it to the list of objects for TraceFunc to GC when it gets
                    // a chance to run next time.
                    _process.GetDataItem<PyObjectAllocator>().QueueForDecRef(oldObj);
                }

                newObj.ob_refcnt.Increment();
                proxy.Write(newObj);
            } else {
                errorText = "Only boolean, numeric or string literals and None are supported.";
            }
        }
Beispiel #3
0
        internal Task Analyze() {
            if (_updater != null) {
                _updater.UpdateStatus(_progressOffset, _progressTotal, "Starting analysis");
            }

            if (!string.IsNullOrEmpty(_logDiagnostic) && AnalysisLog.Output == null) {
                try {
                    AnalysisLog.Output = new StreamWriter(new FileStream(_logDiagnostic, FileMode.Create, FileAccess.Write, FileShare.Read), Encoding.UTF8);
                    AnalysisLog.AsCSV = _logDiagnostic.EndsWith(".csv", StringComparison.InvariantCultureIgnoreCase);
                } catch (Exception ex) {
                    TraceWarning("Failed to open \"{0}\" for logging{1}{2}", _logDiagnostic, Environment.NewLine, ex.ToString());
                }
            }

            var callDepthOverrides = GetCallDepthOverrides();
            var skipModules = new HashSet<string>(GetSkipModules(), StringComparer.Ordinal);

            foreach (var files in _analyzeFileGroups) {
                if (_cancel.IsCancellationRequested) {
                    break;
                }

                if (files.Count == 0) {
                    continue;
                }

                var outDir = GetOutputDir(files[0]);

                if (_dryRun) {
                    foreach (var file in files) {
                        if (ContainsModule(skipModules, file.ModuleName)) {
                            continue;
                        }

                        Debug.Assert(!file.IsCompiled);
                        var idbFile = PathUtils.CreateFriendlyDirectoryPath(
                            _outDir,
                            Path.Combine(outDir, file.ModuleName)
                        );
                        TraceDryRun("ANALYZE;{0};{1}.idb", file.SourceFile, idbFile);
                    }
                    continue;
                }

                Directory.CreateDirectory(outDir);

                TraceInformation("Start group \"{0}\" with {1} files", files[0].LibraryPath, files.Count);
                AnalysisLog.StartFileGroup(files[0].LibraryPath, files.Count);
                Console.WriteLine("Now analyzing: {0}", files[0].LibraryPath);
                string currentLibrary;
                if (_treatPathsAsStandardLibrary.Contains(files[0].LibraryPath)) {
                    currentLibrary = "standard library";
                } else {
                    currentLibrary = PathUtils.GetFileOrDirectoryName(files[0].LibraryPath);
                }

                using (var factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(
                    _version,
                    null,
                    new[] { _outDir, outDir }.Concat(_baseDb.Skip(1)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray()
                ))
                using (var projectState = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions()) {
                    int? mostItemsInQueue = null;
                    if (_updater != null) {
                        projectState.SetQueueReporting(itemsInQueue => {
                            if (itemsInQueue > (mostItemsInQueue ?? 0)) {
                                mostItemsInQueue = itemsInQueue;
                            }

                            if (mostItemsInQueue > 0) {
                                var progress = (files.Count * (mostItemsInQueue - itemsInQueue)) / mostItemsInQueue;
                                _updater.UpdateStatus(_progressOffset + (progress ?? 0), _progressTotal,
                                    "Analyzing " + currentLibrary);
                            } else {
                                _updater.UpdateStatus(_progressOffset + files.Count, _progressTotal,
                                    "Analyzing " + currentLibrary);
                            }
                        }, 10);
                    }

                    try {
                        using (var key = Registry.CurrentUser.OpenSubKey(AnalysisLimitsKey)) {
                            projectState.Limits = AnalysisLimits.LoadFromStorage(key, defaultToStdLib: true);
                        }
                    } catch (SecurityException) {
                        projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
                    } catch (UnauthorizedAccessException) {
                        projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
                    } catch (IOException) {
                        projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
                    }

                    var items = files.Select(f => new AnalysisItem(f)).ToList();

                    foreach (var item in items) {
                        if (_cancel.IsCancellationRequested) {
                            break;
                        }

                        item.Entry = projectState.AddModule(item.ModuleName, item.SourceFile);

                        foreach (var name in ModulePath.GetParents(item.ModuleName, includeFullName: true)) {
                            int depth;
                            if (callDepthOverrides.TryGetValue(name, out depth)) {
                                TraceVerbose("Set CallDepthLimit to 0 for {0}", item.ModuleName);
                                item.Entry.Properties[AnalysisLimits.CallDepthKey] = depth;
                                break;
                            }
                        }
                    }

                    foreach (var item in items) {
                        if (_cancel.IsCancellationRequested) {
                            break;
                        }

                        if (ContainsModule(skipModules, item.ModuleName)) {
                            continue;
                        }

                        if (_updater != null) {
                            _updater.UpdateStatus(_progressOffset, _progressTotal,
                                string.Format("Parsing {0}", currentLibrary));
                        }
                        try {
                            var sourceUnit = new FileStream(item.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read);
                            var errors = new CollectingErrorSink();
                            var opts = new ParserOptions() { BindReferences = true, ErrorSink = errors };

                            TraceInformation("Parsing \"{0}\" (\"{1}\")", item.ModuleName, item.SourceFile);
                            item.Tree = Parser.CreateParser(sourceUnit, _version.ToLanguageVersion(), opts).ParseFile();
                            if (errors.Errors.Any() || errors.Warnings.Any()) {
                                TraceWarning("File \"{0}\" contained parse errors", item.SourceFile);
                                TraceInformation(string.Join(Environment.NewLine, errors.Errors.Concat(errors.Warnings)
                                    .Select(er => string.Format("{0} {1}", er.Span, er.Message))));
                            }
                        } catch (Exception ex) {
                            TraceError("Error parsing \"{0}\" \"{1}\"{2}{3}", item.ModuleName, item.SourceFile, Environment.NewLine, ex.ToString());
                        }
                    }

                    TraceInformation("Parsing complete");

                    foreach (var item in items) {
                        if (_cancel.IsCancellationRequested) {
                            break;
                        }

                        if (item.Tree != null) {
                            item.Entry.UpdateTree(item.Tree, null);
                        }
                    }

                    foreach (var item in items) {
                        if (_cancel.IsCancellationRequested) {
                            break;
                        }

                        try {
                            if (item.Tree != null) {
                                TraceInformation("Analyzing \"{0}\"", item.ModuleName);
                                item.Entry.Analyze(_cancel, true);
                                TraceVerbose("Analyzed \"{0}\"", item.SourceFile);
                            }
                        } catch (Exception ex) {
                            TraceError("Error analyzing \"{0}\" \"{1}\"{2}{3}", item.ModuleName, item.SourceFile, Environment.NewLine, ex.ToString());
                        }
                    }

                    if (items.Count > 0 && !_cancel.IsCancellationRequested) {
                        TraceInformation("Starting analysis of {0} modules", items.Count);
                        items[0].Entry.AnalysisGroup.AnalyzeQueuedEntries(_cancel);
                        TraceInformation("Analysis complete");
                    }

                    if (_cancel.IsCancellationRequested) {
                        break;
                    }

                    TraceInformation("Saving group \"{0}\"", files[0].LibraryPath);
                    if (_updater != null) {
                        _progressOffset += files.Count;
                        _updater.UpdateStatus(_progressOffset, _progressTotal, "Saving " + currentLibrary);
                    }
                    Directory.CreateDirectory(outDir);
                    new SaveAnalysis().Save(projectState, outDir);
                    TraceInformation("End of group \"{0}\"", files[0].LibraryPath);
                    AnalysisLog.EndFileGroup();

                    AnalysisLog.Flush();
                }
            }

            // Lets us have an awaitable function, even though it doesn't need
            // to be async yet. This helps keep the interfaces consistent.
            return Task.FromResult<object>(null);
        }
Beispiel #4
0
        /// <summary>
        /// Gets the current AST and the code string for the project entry and returns the current code.
        /// </summary>
        public static PythonAst GetVerbatimAstAndCode(this IPythonProjectEntry projectFile, PythonLanguageVersion langVersion, int bufferId, out int version, out string code) {
            ParserOptions options = new ParserOptions { BindReferences = true, Verbatim = true };

            code = projectFile.GetCurrentCode(bufferId, out version);
            if (code != null) {
                var parser = Parser.CreateParser(
                    new StringReader(code),
                    langVersion,
                    options
                );

                return parser.ParseFile();
            }
            return null;
        }
Beispiel #5
0
        /// <summary>
        /// Tries to evaluate the given expression by treating it as a chain of member access and indexing operations (e.g. <c>fob[0].oar.baz['abc'].blah</c>),
        /// and looking up the corresponding members in data model provided by <see cref="GetFrameLocals"/>.
        /// </summary>
        /// <param name="vars">List of variables, in the context of which the expression is evaluated.</param>
        /// <returns>
        /// <c>true</c> if evaluation was successful, or if it failed and no fallback is possible (e.g. expression is invalid).
        /// <c>false</c> if evaluation was not successful due to the limitations of this evaluator, and it may be possible to evaluate it correctly by other means.
        /// </returns>
        private bool EvaluateExpressionByWalkingObjects(IEnumerable<DkmSuccessEvaluationResult> vars, DkmInspectionContext inspectionContext, DkmWorkList workList, DkmLanguageExpression expression, DkmStackWalkFrame stackFrame, DkmCompletionRoutine<DkmEvaluateExpressionAsyncResult> completionRoutine) {
            var pyrtInfo = stackFrame.Thread.Process.GetPythonRuntimeInfo();

            var parserOptions = new ParserOptions { ErrorSink = new StringErrorSink() };
            var parser = Parser.CreateParser(new StringReader(expression.Text), pyrtInfo.LanguageVersion, parserOptions);

            var expr = ((ReturnStatement)parser.ParseTopExpression().Body).Expression;
            string errorText = parserOptions.ErrorSink.ToString();
            if (!string.IsNullOrEmpty(errorText)) {
                completionRoutine(new DkmEvaluateExpressionAsyncResult(DkmFailedEvaluationResult.Create(
                    inspectionContext, stackFrame, expression.Text, expression.Text,
                    errorText, DkmEvaluationResultFlags.Invalid, null)));
                return true;
            }

            // Unroll the AST into a sequence of member access and indexing operations, if possible.
            var path = new Stack<string>();
            var reprBuilder = new ReprBuilder(new ReprOptions(stackFrame.Thread.Process));
            while (true) {
                var memberExpr = expr as MemberExpression;
                if (memberExpr != null) {
                    path.Push(memberExpr.Name);
                    expr = memberExpr.Target;
                    continue;
                }

                var indexingExpr = expr as IndexExpression;
                if (indexingExpr != null) {
                    var indexExpr = indexingExpr.Index as ConstantExpression;
                    if (indexExpr != null) {
                        reprBuilder.Clear();
                        reprBuilder.AppendFormat("[{0:PY}]", indexExpr.Value);
                        path.Push(reprBuilder.ToString());
                        expr = indexingExpr.Target;
                        continue;
                    }
                }

                break;
            }

            var varExpr = expr as NameExpression;
            if (varExpr == null) {
                return false;
            }
            path.Push(varExpr.Name);

            // Walk the path through Locals
            while (true) {
                var name = path.Pop();

                var evalResult = vars.FirstOrDefault(er => er.Name == name);
                if (evalResult == null) {
                    return false;
                }

                if (path.Count == 0) {
                    // Clone the evaluation result, but use expression text as its name.
                    DkmDataItem dataItem =
                        (DkmDataItem)evalResult.GetDataItem<PyObjectEvaluationResult>() ??
                        (DkmDataItem)evalResult.GetDataItem<GlobalsEvaluationResult>() ??
                        (DkmDataItem)evalResult.GetDataItem<CppViewEvaluationResult>() ??
                        (DkmDataItem)evalResult.GetDataItem<RawEvaluationResult>();
                    evalResult = DkmSuccessEvaluationResult.Create(
                        evalResult.InspectionContext,
                        evalResult.StackFrame,
                        expression.Text,
                        expression.Text,
                        evalResult.Flags,
                        evalResult.Value,
                        evalResult.EditableValue,
                        evalResult.Type,
                        evalResult.Category,
                        evalResult.Access,
                        evalResult.StorageType,
                        evalResult.TypeModifierFlags,
                        evalResult.Address,
                        evalResult.CustomUIVisualizers,
                        evalResult.ExternalModules,
                        dataItem);

                    completionRoutine(new DkmEvaluateExpressionAsyncResult(evalResult));
                    return true;
                }

                var childWorkList = DkmWorkList.Create(null);
                evalResult.GetChildren(childWorkList, 0, inspectionContext, getChildrenResult =>
                    getChildrenResult.EnumContext.GetItems(childWorkList, 0, int.MaxValue, getItemsResult =>
                        vars = getItemsResult.Items.OfType<DkmSuccessEvaluationResult>()));
                childWorkList.Execute();
            }
        }
Beispiel #6
0
        public static Parser CreateParser(TextReader reader, PythonLanguageVersion version, ParserOptions parserOptions) {
            if (reader == null) {
                throw new ArgumentNullException("reader");
            }

            var options = parserOptions ?? ParserOptions.Default;

            Parser parser = null;
            var tokenizer = new Tokenizer(
                version, options.ErrorSink,
                (options.Verbatim ? TokenizerOptions.Verbatim : TokenizerOptions.None) | TokenizerOptions.GroupingRecovery,
                (span, text) => options.RaiseProcessComment(parser, new CommentEventArgs(span, text)));
            tokenizer.Initialize(null, reader, SourceLocation.MinValue);
            tokenizer.IndentationInconsistencySeverity = options.IndentationInconsistencySeverity;

            parser = new Parser(
                tokenizer,
                options.ErrorSink ?? ErrorSink.Null,
                version,
                options.Verbatim,
                options.BindReferences,
                options.PrivatePrefix
            ) { _sourceReader = reader };
            return parser;
        }
Beispiel #7
0
        /// <summary>
        /// Creates a new parser from a seekable stream including scanning the BOM or looking for a # coding: comment to detect the appropriate coding.
        /// </summary>
        public static Parser CreateParser(Stream stream, PythonLanguageVersion version, ParserOptions parserOptions = null) {
            var options = parserOptions ?? ParserOptions.Default;

            var defaultEncoding = version.Is2x() ? PythonAsciiEncoding.Instance : Encoding.UTF8;

            var reader = GetStreamReaderWithEncoding(stream, defaultEncoding, options.ErrorSink);

            return CreateParser(reader, version, options);
        }
Beispiel #8
0
 private ParserOptions MakeParserOptions(CollectingErrorSink errorSink, List<AP.TaskItem> tasks) {
     var options = new ParserOptions {
         ErrorSink = errorSink,
         IndentationInconsistencySeverity = _options.indentation_inconsistency_severity,
         BindReferences = true
     };
     options.ProcessComment += (sender, e) => ProcessComment(tasks, e.Span, e.Text);
     return options;
 }
Beispiel #9
0
 public override Parser CreateParser(PythonLanguageVersion version, ParserOptions options) {
     return Parser.CreateParser(_text, version, options);
 }
Beispiel #10
0
 public abstract Parser CreateParser(PythonLanguageVersion version, ParserOptions options);