private void ParsePythonCode( ITextSnapshot snapshot, TextReader content, Severity indentationSeverity, out PythonAst ast, out CollectingErrorSink errorSink, out List<TaskProviderItem> commentTasks ) { ast = null; errorSink = new CollectingErrorSink(); var tasks = commentTasks = new List<TaskProviderItem>(); var options = new ParserOptions { ErrorSink = errorSink, IndentationInconsistencySeverity = indentationSeverity, BindReferences = true, }; options.ProcessComment += (sender, e) => ProcessComment(tasks, snapshot, e.Span, e.Text); using (var parser = Parser.CreateParser(content, Project.LanguageVersion, options)) { ast = ParseOneFile(ast, parser); } }
public unsafe void SetValueAsString(DkmEvaluationResult result, string value, int timeout, out string errorText) { var pyEvalResult = result.GetDataItem<PyObjectEvaluationResult>(); if (pyEvalResult == null) { Debug.Fail("SetValueAsString called on a DkmEvaluationResult without an associated PyObjectEvaluationResult."); throw new NotSupportedException(); } var proxy = pyEvalResult.ValueStore as IWritableDataProxy; if (proxy == null) { Debug.Fail("SetValueAsString called on a DkmEvaluationResult that does not correspond to an IWritableDataProxy."); throw new InvalidOperationException(); } errorText = null; var process = result.StackFrame.Process; var pyrtInfo = process.GetPythonRuntimeInfo(); var parserOptions = new ParserOptions { ErrorSink = new StringErrorSink() }; var parser = Parser.CreateParser(new StringReader(value), pyrtInfo.LanguageVersion, parserOptions); var body = (ReturnStatement)parser.ParseTopExpression().Body; errorText = parserOptions.ErrorSink.ToString(); if (!string.IsNullOrEmpty(errorText)) { return; } var expr = body.Expression; while (true) { var parenExpr = expr as ParenthesisExpression; if (parenExpr == null) { break; } expr = parenExpr.Expression; } int sign; expr = ForceExplicitSign(expr, out sign); PyObject newObj = null; var constExpr = expr as ConstantExpression; if (constExpr != null) { if (constExpr.Value == null) { newObj = PyObject.None(process); } else if (constExpr.Value is bool) { // In 2.7, 'True' and 'False' are reported as identifiers, not literals, and are handled separately below. newObj = PyBoolObject33.Create(process, (bool)constExpr.Value); } else if (constExpr.Value is string) { if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27) { newObj = PyUnicodeObject27.Create(process, (string)constExpr.Value); } else { newObj = PyUnicodeObject33.Create(process, (string)constExpr.Value); } } else if (constExpr.Value is AsciiString) { newObj = PyBytesObject.Create(process, (AsciiString)constExpr.Value); } } else { var unaryExpr = expr as UnaryExpression; if (unaryExpr != null && sign != 0) { constExpr = unaryExpr.Expression as ConstantExpression; if (constExpr != null) { if (constExpr.Value is BigInteger) { newObj = PyLongObject.Create(process, (BigInteger)constExpr.Value * sign); } else if (constExpr.Value is int) { if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27) { newObj = PyIntObject.Create(process, (int)constExpr.Value * sign); } else { newObj = PyLongObject.Create(process, (int)constExpr.Value * sign); } } else if (constExpr.Value is double) { newObj = PyFloatObject.Create(process, (double)constExpr.Value * sign); } else if (constExpr.Value is Complex) { newObj = PyComplexObject.Create(process, (Complex)constExpr.Value * sign); } } } else { var binExpr = expr as BinaryExpression; if (binExpr != null && (binExpr.Operator == PythonOperator.Add || binExpr.Operator == PythonOperator.Subtract)) { int realSign; var realExpr = ForceExplicitSign(binExpr.Left, out realSign) as UnaryExpression; int imagSign; var imagExpr = ForceExplicitSign(binExpr.Right, out imagSign) as UnaryExpression; if (realExpr != null && realSign != 0 && imagExpr != null && imagSign != 0) { var realConst = realExpr.Expression as ConstantExpression; var imagConst = imagExpr.Expression as ConstantExpression; if (realConst != null && imagConst != null) { var realVal = (realConst.Value as int? ?? realConst.Value as double?) as IConvertible; var imagVal = imagConst.Value as Complex?; if (realVal != null && imagVal != null) { double real = realVal.ToDouble(null) * realSign; double imag = imagVal.Value.Imaginary * imagSign * (binExpr.Operator == PythonOperator.Add ? 1 : -1); newObj = PyComplexObject.Create(process, new Complex(real, imag)); } } } } else { if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27) { // 'True' and 'False' are not literals in 2.x, but we want to treat them as such. var name = expr as NameExpression; if (name != null) { if (name.Name == "True") { newObj = PyBoolObject27.Create(process, true); } else if (name.Name == "False") { newObj = PyBoolObject27.Create(process, false); } } } } } } if (newObj != null) { var oldObj = proxy.Read() as PyObject; if (oldObj != null) { // We can't free the original value without running some code in the process, and it may be holding heap locks. // So don't decrement refcount now, but instead add it to the list of objects for TraceFunc to GC when it gets // a chance to run next time. _process.GetDataItem<PyObjectAllocator>().QueueForDecRef(oldObj); } newObj.ob_refcnt.Increment(); proxy.Write(newObj); } else { errorText = "Only boolean, numeric or string literals and None are supported."; } }
internal Task Analyze() { if (_updater != null) { _updater.UpdateStatus(_progressOffset, _progressTotal, "Starting analysis"); } if (!string.IsNullOrEmpty(_logDiagnostic) && AnalysisLog.Output == null) { try { AnalysisLog.Output = new StreamWriter(new FileStream(_logDiagnostic, FileMode.Create, FileAccess.Write, FileShare.Read), Encoding.UTF8); AnalysisLog.AsCSV = _logDiagnostic.EndsWith(".csv", StringComparison.InvariantCultureIgnoreCase); } catch (Exception ex) { TraceWarning("Failed to open \"{0}\" for logging{1}{2}", _logDiagnostic, Environment.NewLine, ex.ToString()); } } var callDepthOverrides = GetCallDepthOverrides(); var skipModules = new HashSet<string>(GetSkipModules(), StringComparer.Ordinal); foreach (var files in _analyzeFileGroups) { if (_cancel.IsCancellationRequested) { break; } if (files.Count == 0) { continue; } var outDir = GetOutputDir(files[0]); if (_dryRun) { foreach (var file in files) { if (ContainsModule(skipModules, file.ModuleName)) { continue; } Debug.Assert(!file.IsCompiled); var idbFile = PathUtils.CreateFriendlyDirectoryPath( _outDir, Path.Combine(outDir, file.ModuleName) ); TraceDryRun("ANALYZE;{0};{1}.idb", file.SourceFile, idbFile); } continue; } Directory.CreateDirectory(outDir); TraceInformation("Start group \"{0}\" with {1} files", files[0].LibraryPath, files.Count); AnalysisLog.StartFileGroup(files[0].LibraryPath, files.Count); Console.WriteLine("Now analyzing: {0}", files[0].LibraryPath); string currentLibrary; if (_treatPathsAsStandardLibrary.Contains(files[0].LibraryPath)) { currentLibrary = "standard library"; } else { currentLibrary = PathUtils.GetFileOrDirectoryName(files[0].LibraryPath); } using (var factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory( _version, null, new[] { _outDir, outDir }.Concat(_baseDb.Skip(1)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray() )) using (var projectState = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions()) { int? mostItemsInQueue = null; if (_updater != null) { projectState.SetQueueReporting(itemsInQueue => { if (itemsInQueue > (mostItemsInQueue ?? 0)) { mostItemsInQueue = itemsInQueue; } if (mostItemsInQueue > 0) { var progress = (files.Count * (mostItemsInQueue - itemsInQueue)) / mostItemsInQueue; _updater.UpdateStatus(_progressOffset + (progress ?? 0), _progressTotal, "Analyzing " + currentLibrary); } else { _updater.UpdateStatus(_progressOffset + files.Count, _progressTotal, "Analyzing " + currentLibrary); } }, 10); } try { using (var key = Registry.CurrentUser.OpenSubKey(AnalysisLimitsKey)) { projectState.Limits = AnalysisLimits.LoadFromStorage(key, defaultToStdLib: true); } } catch (SecurityException) { projectState.Limits = AnalysisLimits.GetStandardLibraryLimits(); } catch (UnauthorizedAccessException) { projectState.Limits = AnalysisLimits.GetStandardLibraryLimits(); } catch (IOException) { projectState.Limits = AnalysisLimits.GetStandardLibraryLimits(); } var items = files.Select(f => new AnalysisItem(f)).ToList(); foreach (var item in items) { if (_cancel.IsCancellationRequested) { break; } item.Entry = projectState.AddModule(item.ModuleName, item.SourceFile); foreach (var name in ModulePath.GetParents(item.ModuleName, includeFullName: true)) { int depth; if (callDepthOverrides.TryGetValue(name, out depth)) { TraceVerbose("Set CallDepthLimit to 0 for {0}", item.ModuleName); item.Entry.Properties[AnalysisLimits.CallDepthKey] = depth; break; } } } foreach (var item in items) { if (_cancel.IsCancellationRequested) { break; } if (ContainsModule(skipModules, item.ModuleName)) { continue; } if (_updater != null) { _updater.UpdateStatus(_progressOffset, _progressTotal, string.Format("Parsing {0}", currentLibrary)); } try { var sourceUnit = new FileStream(item.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read); var errors = new CollectingErrorSink(); var opts = new ParserOptions() { BindReferences = true, ErrorSink = errors }; TraceInformation("Parsing \"{0}\" (\"{1}\")", item.ModuleName, item.SourceFile); item.Tree = Parser.CreateParser(sourceUnit, _version.ToLanguageVersion(), opts).ParseFile(); if (errors.Errors.Any() || errors.Warnings.Any()) { TraceWarning("File \"{0}\" contained parse errors", item.SourceFile); TraceInformation(string.Join(Environment.NewLine, errors.Errors.Concat(errors.Warnings) .Select(er => string.Format("{0} {1}", er.Span, er.Message)))); } } catch (Exception ex) { TraceError("Error parsing \"{0}\" \"{1}\"{2}{3}", item.ModuleName, item.SourceFile, Environment.NewLine, ex.ToString()); } } TraceInformation("Parsing complete"); foreach (var item in items) { if (_cancel.IsCancellationRequested) { break; } if (item.Tree != null) { item.Entry.UpdateTree(item.Tree, null); } } foreach (var item in items) { if (_cancel.IsCancellationRequested) { break; } try { if (item.Tree != null) { TraceInformation("Analyzing \"{0}\"", item.ModuleName); item.Entry.Analyze(_cancel, true); TraceVerbose("Analyzed \"{0}\"", item.SourceFile); } } catch (Exception ex) { TraceError("Error analyzing \"{0}\" \"{1}\"{2}{3}", item.ModuleName, item.SourceFile, Environment.NewLine, ex.ToString()); } } if (items.Count > 0 && !_cancel.IsCancellationRequested) { TraceInformation("Starting analysis of {0} modules", items.Count); items[0].Entry.AnalysisGroup.AnalyzeQueuedEntries(_cancel); TraceInformation("Analysis complete"); } if (_cancel.IsCancellationRequested) { break; } TraceInformation("Saving group \"{0}\"", files[0].LibraryPath); if (_updater != null) { _progressOffset += files.Count; _updater.UpdateStatus(_progressOffset, _progressTotal, "Saving " + currentLibrary); } Directory.CreateDirectory(outDir); new SaveAnalysis().Save(projectState, outDir); TraceInformation("End of group \"{0}\"", files[0].LibraryPath); AnalysisLog.EndFileGroup(); AnalysisLog.Flush(); } } // Lets us have an awaitable function, even though it doesn't need // to be async yet. This helps keep the interfaces consistent. return Task.FromResult<object>(null); }
/// <summary> /// Gets the current AST and the code string for the project entry and returns the current code. /// </summary> public static PythonAst GetVerbatimAstAndCode(this IPythonProjectEntry projectFile, PythonLanguageVersion langVersion, int bufferId, out int version, out string code) { ParserOptions options = new ParserOptions { BindReferences = true, Verbatim = true }; code = projectFile.GetCurrentCode(bufferId, out version); if (code != null) { var parser = Parser.CreateParser( new StringReader(code), langVersion, options ); return parser.ParseFile(); } return null; }
/// <summary> /// Tries to evaluate the given expression by treating it as a chain of member access and indexing operations (e.g. <c>fob[0].oar.baz['abc'].blah</c>), /// and looking up the corresponding members in data model provided by <see cref="GetFrameLocals"/>. /// </summary> /// <param name="vars">List of variables, in the context of which the expression is evaluated.</param> /// <returns> /// <c>true</c> if evaluation was successful, or if it failed and no fallback is possible (e.g. expression is invalid). /// <c>false</c> if evaluation was not successful due to the limitations of this evaluator, and it may be possible to evaluate it correctly by other means. /// </returns> private bool EvaluateExpressionByWalkingObjects(IEnumerable<DkmSuccessEvaluationResult> vars, DkmInspectionContext inspectionContext, DkmWorkList workList, DkmLanguageExpression expression, DkmStackWalkFrame stackFrame, DkmCompletionRoutine<DkmEvaluateExpressionAsyncResult> completionRoutine) { var pyrtInfo = stackFrame.Thread.Process.GetPythonRuntimeInfo(); var parserOptions = new ParserOptions { ErrorSink = new StringErrorSink() }; var parser = Parser.CreateParser(new StringReader(expression.Text), pyrtInfo.LanguageVersion, parserOptions); var expr = ((ReturnStatement)parser.ParseTopExpression().Body).Expression; string errorText = parserOptions.ErrorSink.ToString(); if (!string.IsNullOrEmpty(errorText)) { completionRoutine(new DkmEvaluateExpressionAsyncResult(DkmFailedEvaluationResult.Create( inspectionContext, stackFrame, expression.Text, expression.Text, errorText, DkmEvaluationResultFlags.Invalid, null))); return true; } // Unroll the AST into a sequence of member access and indexing operations, if possible. var path = new Stack<string>(); var reprBuilder = new ReprBuilder(new ReprOptions(stackFrame.Thread.Process)); while (true) { var memberExpr = expr as MemberExpression; if (memberExpr != null) { path.Push(memberExpr.Name); expr = memberExpr.Target; continue; } var indexingExpr = expr as IndexExpression; if (indexingExpr != null) { var indexExpr = indexingExpr.Index as ConstantExpression; if (indexExpr != null) { reprBuilder.Clear(); reprBuilder.AppendFormat("[{0:PY}]", indexExpr.Value); path.Push(reprBuilder.ToString()); expr = indexingExpr.Target; continue; } } break; } var varExpr = expr as NameExpression; if (varExpr == null) { return false; } path.Push(varExpr.Name); // Walk the path through Locals while (true) { var name = path.Pop(); var evalResult = vars.FirstOrDefault(er => er.Name == name); if (evalResult == null) { return false; } if (path.Count == 0) { // Clone the evaluation result, but use expression text as its name. DkmDataItem dataItem = (DkmDataItem)evalResult.GetDataItem<PyObjectEvaluationResult>() ?? (DkmDataItem)evalResult.GetDataItem<GlobalsEvaluationResult>() ?? (DkmDataItem)evalResult.GetDataItem<CppViewEvaluationResult>() ?? (DkmDataItem)evalResult.GetDataItem<RawEvaluationResult>(); evalResult = DkmSuccessEvaluationResult.Create( evalResult.InspectionContext, evalResult.StackFrame, expression.Text, expression.Text, evalResult.Flags, evalResult.Value, evalResult.EditableValue, evalResult.Type, evalResult.Category, evalResult.Access, evalResult.StorageType, evalResult.TypeModifierFlags, evalResult.Address, evalResult.CustomUIVisualizers, evalResult.ExternalModules, dataItem); completionRoutine(new DkmEvaluateExpressionAsyncResult(evalResult)); return true; } var childWorkList = DkmWorkList.Create(null); evalResult.GetChildren(childWorkList, 0, inspectionContext, getChildrenResult => getChildrenResult.EnumContext.GetItems(childWorkList, 0, int.MaxValue, getItemsResult => vars = getItemsResult.Items.OfType<DkmSuccessEvaluationResult>())); childWorkList.Execute(); } }
public static Parser CreateParser(TextReader reader, PythonLanguageVersion version, ParserOptions parserOptions) { if (reader == null) { throw new ArgumentNullException("reader"); } var options = parserOptions ?? ParserOptions.Default; Parser parser = null; var tokenizer = new Tokenizer( version, options.ErrorSink, (options.Verbatim ? TokenizerOptions.Verbatim : TokenizerOptions.None) | TokenizerOptions.GroupingRecovery, (span, text) => options.RaiseProcessComment(parser, new CommentEventArgs(span, text))); tokenizer.Initialize(null, reader, SourceLocation.MinValue); tokenizer.IndentationInconsistencySeverity = options.IndentationInconsistencySeverity; parser = new Parser( tokenizer, options.ErrorSink ?? ErrorSink.Null, version, options.Verbatim, options.BindReferences, options.PrivatePrefix ) { _sourceReader = reader }; return parser; }
/// <summary> /// Creates a new parser from a seekable stream including scanning the BOM or looking for a # coding: comment to detect the appropriate coding. /// </summary> public static Parser CreateParser(Stream stream, PythonLanguageVersion version, ParserOptions parserOptions = null) { var options = parserOptions ?? ParserOptions.Default; var defaultEncoding = version.Is2x() ? PythonAsciiEncoding.Instance : Encoding.UTF8; var reader = GetStreamReaderWithEncoding(stream, defaultEncoding, options.ErrorSink); return CreateParser(reader, version, options); }
private ParserOptions MakeParserOptions(CollectingErrorSink errorSink, List<AP.TaskItem> tasks) { var options = new ParserOptions { ErrorSink = errorSink, IndentationInconsistencySeverity = _options.indentation_inconsistency_severity, BindReferences = true }; options.ProcessComment += (sender, e) => ProcessComment(tasks, e.Span, e.Text); return options; }
public override Parser CreateParser(PythonLanguageVersion version, ParserOptions options) { return Parser.CreateParser(_text, version, options); }
public abstract Parser CreateParser(PythonLanguageVersion version, ParserOptions options);