private static void ValidateLocations(SourceLocation start, SourceLocation end) { if (start.IsValid && end.IsValid) { if (start > end) { throw new ArgumentException("Start and End must be well ordered"); } } else { if (start.IsValid || end.IsValid) { throw new ArgumentException("Start and End must both be valid or both invalid"); } } }
/// <summary> /// Evaluates the given expression in at the provided line number and returns the values /// that the expression can evaluate to. /// </summary> /// <param name="exprText">The expression to determine the result of.</param> /// <param name="location">The location in the file where the expression should be evaluated.</param> /// <remarks>New in 2.2</remarks> public IEnumerable<AnalysisValue> GetValues(string exprText, SourceLocation location) { var scope = FindScope(location); var privatePrefix = GetPrivatePrefixClassName(scope); var expr = Statement.GetExpression(GetAstFromText(exprText, privatePrefix).Body); var unit = GetNearestEnclosingAnalysisUnit(scope); var eval = new ExpressionEvaluator(unit.CopyForEval(), scope, mergeScopes: true); var values = eval.Evaluate(expr); var res = AnalysisSet.EmptyUnion; foreach (var v in values) { MultipleMemberInfo multipleMembers = v as MultipleMemberInfo; if (multipleMembers != null) { foreach (var member in multipleMembers.Members) { if (member.IsCurrent) { res = res.Add(member); } } } else if (v.IsCurrent) { res = res.Add(v); } } return res; }
/// <summary> /// Evaluates a given expression and returns a list of members which /// exist in the expression. /// /// If the expression is an empty string returns all available members /// at that location. /// </summary> /// <param name="exprText">The expression to find members for.</param> /// </param> /// <param name="location"> /// The location in the file where the expression should be evaluated. /// </param> /// <remarks>New in 2.2</remarks> public IEnumerable<MemberResult> GetMembers( string exprText, SourceLocation location, GetMemberOptions options = GetMemberOptions.IntersectMultipleResults ) { if (exprText.Length == 0) { return GetAllAvailableMembers(location, options); } var scope = FindScope(location); var privatePrefix = GetPrivatePrefixClassName(scope); var expr = Statement.GetExpression(GetAstFromText(exprText, privatePrefix).Body); if (expr is ConstantExpression && ((ConstantExpression)expr).Value is int) { // no completions on integer ., the user is typing a float return Enumerable.Empty<MemberResult>(); } var errorWalker = new ErrorWalker(); expr.Walk(errorWalker); if (errorWalker.HasError) { return null; } var unit = GetNearestEnclosingAnalysisUnit(scope); var eval = new ExpressionEvaluator(unit.CopyForEval(), scope, mergeScopes: true); IAnalysisSet lookup; if (options.HasFlag(GetMemberOptions.NoMemberRecursion)) { lookup = eval.EvaluateNoMemberRecursion(expr); } else { lookup = eval.Evaluate(expr); } return GetMemberResults(lookup, scope, options); }
/// <summary> /// Gets the hierarchy of class and function definitions at the /// specified location. /// </summary> /// <param name="location">The location in the file.</param> /// <remarks>New in 2.2</remarks> public IEnumerable<MemberResult> GetDefinitionTree(SourceLocation location) { try { return FindScope(location).EnumerateTowardsGlobal .Select(s => new MemberResult(s.Name, s.GetMergedAnalysisValues())) .ToList(); } catch (Exception) { // TODO: log exception Debug.Fail("Failed to find scope. Bad state in analysis"); return new[] { new MemberResult("Unknown", new AnalysisValue[] { }) }; } }
/// <summary> /// Gets the AST for the given text as if it appeared at the specified /// location. /// /// If the expression is a member expression such as "fob.__bar" and the /// line number is inside of a class definition this will return a /// MemberExpression with the mangled name like "fob._ClassName__bar". /// </summary> /// <param name="exprText">The expression to evaluate.</param> /// <param name="index"> /// The 0-based index into the file where the expression should be /// evaluated. /// </param> /// <remarks>New in 2.2</remarks> public PythonAst GetAstFromText(string exprText, SourceLocation location) { var scopes = FindScope(location); var privatePrefix = GetPrivatePrefixClassName(scopes); return GetAstFromText(exprText, privatePrefix); }
/// <summary> /// Constructs a new span with a specific start and end location. /// </summary> /// <param name="start">The beginning of the span.</param> /// <param name="end">The end of the span.</param> public SourceSpan(SourceLocation start, SourceLocation end) { ValidateLocations(start, end); this._start = start; this._end = end; }
private static bool IsInFunctionParameter(InterpreterScope scope, PythonAst tree, SourceLocation location) { var function = scope.Node as FunctionDefinition; if (function == null) { // Not a function return false; } if (location.Index < function.StartIndex || location.Index >= function.Body.StartIndex) { // Not within the def line return false; } return function.Parameters != null && function.Parameters.Any(p => { var paramName = p.GetVerbatimImage(tree) ?? p.Name; return location.Index >= p.StartIndex && location.Index <= p.StartIndex + paramName.Length; }); }
private static InterpreterScope FindScope(InterpreterScope parent, PythonAst tree, SourceLocation location) { var children = parent.Children.Where(c => !(c is StatementScope)).ToList(); InterpreterScope candidate = null; for (int i = 0; i < children.Count; ++i) { if (IsInFunctionParameter(children[i], tree, location)) { // In parameter name scope, so consider the function scope. candidate = children[i]; continue; } int start = children[i].GetBodyStart(tree); if (start > location.Index) { // We've gone past index completely so our last candidate is // the best one. break; } int end = children[i].GetStop(tree); if (i + 1 < children.Count) { int nextStart = children[i + 1].GetStart(tree); if (nextStart > end) { end = nextStart; } } if (location.Index <= end || (candidate == null && i + 1 == children.Count)) { candidate = children[i]; } } if (candidate == null) { // No children, so we must belong in our parent return parent; } int scopeIndent = GetParentScopeIndent(candidate, tree); if (location.Column <= scopeIndent) { // Candidate is at deeper indentation than location and the // candidate is scoped, so return the parent instead. return parent; } // Recurse to check children of candidate scope var child = FindScope(candidate, tree, location); var funcChild = child as FunctionScope; if (funcChild != null && funcChild.Function.FunctionDefinition.IsLambda && child.GetStop(tree) < location.Index) { // Do not want to extend a lambda function's scope to the end of // the parent scope. return parent; } return child; }
/// <summary> /// Gets the variables the given expression evaluates to. Variables /// include parameters, locals, and fields assigned on classes, modules /// and instances. /// /// Variables are classified as either definitions or references. Only /// parameters have unique definition points - all other types of /// variables have only one or more references. /// </summary> /// <param name="exprText">The expression to find variables for.</param> /// <param name="location"> /// The location in the file where the expression should be evaluated. /// </param> /// <remarks>New in 2.2</remarks> public VariablesResult GetVariables(string exprText, SourceLocation location) { var scope = FindScope(location); string privatePrefix = GetPrivatePrefixClassName(scope); var ast = GetAstFromText(exprText, privatePrefix); var expr = Statement.GetExpression(ast.Body); var unit = GetNearestEnclosingAnalysisUnit(scope); NameExpression name = expr as NameExpression; IEnumerable<IAnalysisVariable> variables = Enumerable.Empty<IAnalysisVariable>(); if (name != null) { var defScope = scope.EnumerateTowardsGlobal.FirstOrDefault(s => s.ContainsVariable(name.Name) && (s == scope || s.VisibleToChildren || IsFirstLineOfFunction(scope, s, location))); if (defScope == null) { variables = _unit.ProjectState.BuiltinModule.GetDefinitions(name.Name) .SelectMany(ToVariables); } else { variables = GetVariablesInScope(name, defScope).Distinct(); } } else { MemberExpression member = expr as MemberExpression; if (member != null && !string.IsNullOrEmpty(member.Name)) { var eval = new ExpressionEvaluator(unit.CopyForEval(), scope, mergeScopes: true); var objects = eval.Evaluate(member.Target); foreach (var v in objects) { var container = v as IReferenceableContainer; if (container != null) { variables = ReferencablesToVariables(container.GetDefinitions(member.Name)); break; } } } } return new VariablesResult(variables, ast); }
/// <summary> /// Gets the hierarchy of class and function definitions at the /// specified location. /// </summary> /// <param name="location">The location in the file.</param> /// <remarks>New in 2.2</remarks> public IEnumerable<MemberResult> GetDefinitionTree(SourceLocation location) { try { return FindScope(location).EnumerateTowardsGlobal .Select(s => new MemberResult(s.Name, s.GetMergedAnalysisValues())) .ToList(); } catch (Exception) { // TODO: log exception return new[] { new MemberResult("Unknown", null) }; } }
private bool MarkCoverage(bool inheritCoverage, SourceLocation start, SourceLocation end, bool isCovered) { bool covered; int line = start.Line; CoverageLineInfo info; if (!CurScope.Lines.TryGetValue(line, out info)) { CurScope.Lines[line] = info = new CoverageLineInfo(); } info.Covered = covered = inheritCoverage && (_blockCovered ?? false) || isCovered; info.ColumnStart = Math.Min(info.ColumnStart, start.Column); info.ColumnEnd = Math.Max(info.ColumnEnd, end.Column); return covered; }
/// <summary> /// Compares two specified location values. /// </summary> /// <param name="left">One location to compare.</param> /// <param name="right">The other location to compare.</param> /// <returns>0 if the locations are equal, -1 if the left one is less than the right one, 1 otherwise.</returns> public static int Compare(SourceLocation left, SourceLocation right) { if (left < right) return -1; if (right > left) return 1; return 0; }
public DkmStackWalkFrame[] FilterNextFrame(DkmStackContext stackContext, DkmStackWalkFrame nativeFrame) { PyFrameObject pythonFrame = null; var nativeModuleInstance = nativeFrame.ModuleInstance; if (nativeModuleInstance == _pyrtInfo.DLLs.DebuggerHelper) { if (_pyrtInfo.LanguageVersion < PythonLanguageVersion.V36 || (pythonFrame = PyFrameObject.TryCreate(nativeFrame)) == null) { return DebuggerOptions.ShowNativePythonFrames ? new[] { nativeFrame } : new DkmStackWalkFrame[0]; } } var result = new List<DkmStackWalkFrame>(); if (pythonFrame == null) { var stackWalkData = stackContext.GetDataItem<StackWalkContextData>(); if (stackWalkData == null) { stackWalkData = new StackWalkContextData(); stackContext.SetDataItem(DkmDataCreationDisposition.CreateNew, stackWalkData); } bool? wasLastFrameNative = stackWalkData.IsLastFrameNative; if (nativeModuleInstance != _pyrtInfo.DLLs.Python && nativeModuleInstance != _pyrtInfo.DLLs.CTypes) { stackWalkData.IsLastFrameNative = true; if (wasLastFrameNative == false) { result.Add(DkmStackWalkFrame.Create(nativeFrame.Thread, null, nativeFrame.FrameBase, nativeFrame.FrameSize, DkmStackWalkFrameFlags.NonuserCode, "[Native to Python Transition]", null, null)); } else { stackWalkData.IsLastFrameNative = true; } result.Add(nativeFrame); return result.ToArray(); } else { stackWalkData.IsLastFrameNative = false; if (wasLastFrameNative == true) { result.Add(DkmStackWalkFrame.Create(nativeFrame.Thread, null, nativeFrame.FrameBase, nativeFrame.FrameSize, DkmStackWalkFrameFlags.NonuserCode, "[Python to Native Transition]", null, null)); } } pythonFrame = PyFrameObject.TryCreate(nativeFrame); } if (pythonFrame == null) { if (DebuggerOptions.ShowNativePythonFrames) { result.Add(nativeFrame); } return result.ToArray(); } PyCodeObject code = pythonFrame.f_code.Read(); var loc = new SourceLocation( code.co_filename.Read().ToStringOrNull(), pythonFrame.f_lineno.Read(), code.co_name.Read().ToStringOrNull(), nativeFrame.InstructionAddress as DkmNativeInstructionAddress); var pythonRuntime = _process.GetPythonRuntimeInstance(); var pythonModuleInstances = pythonRuntime.GetModuleInstances().OfType<DkmCustomModuleInstance>(); var pyModuleInstance = pythonModuleInstances.Where(m => m.FullName == loc.FileName).FirstOrDefault(); if (pyModuleInstance == null) { pyModuleInstance = pythonModuleInstances.Single(m => m.Module.Id.Mvid == Guids.UnknownPythonModuleGuid); } var encodedLocation = loc.Encode(); var instrAddr = DkmCustomInstructionAddress.Create(pythonRuntime, pyModuleInstance, encodedLocation, 0, encodedLocation, null); var frame = DkmStackWalkFrame.Create( nativeFrame.Thread, instrAddr, nativeFrame.FrameBase, nativeFrame.FrameSize, DkmStackWalkFrameFlags.None, null, nativeFrame.Registers, nativeFrame.Annotations); result.Add(frame); if (DebuggerOptions.ShowNativePythonFrames) { result.Add(nativeFrame); } return result.ToArray(); }
public void GetFrameName(DkmInspectionContext inspectionContext, DkmWorkList workList, DkmStackWalkFrame frame, DkmVariableInfoFlags argumentFlags, DkmCompletionRoutine<DkmGetFrameNameAsyncResult> completionRoutine) { var insAddr = frame.InstructionAddress as DkmCustomInstructionAddress; if (insAddr == null) { Debug.Fail("GetFrameName called on a Python frame without a proper instruction address."); throw new InvalidOperationException(); } var loc = new SourceLocation(insAddr.AdditionalData, frame.Process); completionRoutine(new DkmGetFrameNameAsyncResult(loc.FunctionName)); }
/// <summary> /// Gets information about methods defined on base classes but not /// directly on the current class. /// </summary> /// <param name="location">The location in the file.</param> /// <remarks>New in 2.2</remarks> public IEnumerable<IOverloadResult> GetOverrideable(SourceLocation location) { try { var result = new List<IOverloadResult>(); var scope = FindScope(location); var cls = scope as ClassScope; if (cls == null) { return result; } var handled = new HashSet<string>(cls.Children.Select(child => child.Name)); var cls2 = scope as ClassScope; var mro = (cls2 ?? cls).Class.Mro; if (mro == null) { return result; } foreach (var baseClass in mro.Skip(1).SelectMany()) { ClassInfo klass; BuiltinClassInfo builtinClass; IEnumerable<AnalysisValue> source; if ((klass = baseClass as ClassInfo) != null) { source = klass.Scope.Children .Where(child => child != null && child.AnalysisValue != null) .Select(child => child.AnalysisValue); } else if ((builtinClass = baseClass as BuiltinClassInfo) != null) { source = builtinClass.GetAllMembers(InterpreterContext) .SelectMany(kv => kv.Value) .Where(child => child != null && (child.MemberType == PythonMemberType.Function || child.MemberType == PythonMemberType.Method)); } else { continue; } foreach (var child in source) { if (!child.Overloads.Any()) { continue; } try { var overload = child.Overloads.Aggregate( (best, o) => o.Parameters.Length > best.Parameters.Length ? o : best ); if (handled.Contains(overload.Name)) { continue; } handled.Add(overload.Name); result.Add(overload); } catch { // TODO: log exception // Exceptions only affect the current override. Others may still be offerred. } } } return result; } catch (Exception) { // TODO: log exception return new IOverloadResult[0]; } }
private Response GetOverrides(AP.OverridesCompletionRequest request) { var projectFile = _projectFiles[request.fileId] as IPythonProjectEntry; var analysis = projectFile.Analysis; if (analysis != null) { var location = new SourceLocation(request.index, request.line, request.column); var cls = analysis.GetDefinitionTree(location).LastOrDefault(member => member.MemberType == PythonMemberType.Class); var members = analysis.GetOverrideable(location).ToArray(); return new AP.OverridesCompletionResponse() { overrides = members .Select(member => new AP.Override() { name = member.Name, doc = member.Documentation, completion = MakeCompletionString(request, member, cls.Name) }).ToArray() }; } return new AP.OverridesCompletionResponse() { overrides = Array.Empty<AP.Override>() }; }
/// <summary> /// Gets information about the available signatures for the given expression. /// </summary> /// <param name="exprText">The expression to get signatures for.</param> /// <param name="location">The location in the file.</param> /// <remarks>New in 2.2</remarks> public IEnumerable<IOverloadResult> GetSignatures(string exprText, SourceLocation location) { try { var scope = FindScope(location); var unit = GetNearestEnclosingAnalysisUnit(scope); var eval = new ExpressionEvaluator(unit.CopyForEval(), scope, mergeScopes: true); using (var parser = Parser.CreateParser(new StringReader(exprText), _unit.ProjectState.LanguageVersion)) { var expr = GetExpression(parser.ParseTopExpression().Body); if (expr is ListExpression || expr is TupleExpression || expr is DictionaryExpression) { return Enumerable.Empty<IOverloadResult>(); } var lookup = eval.Evaluate(expr); lookup = AnalysisSet.Create(lookup.Where(av => !(av is MultipleMemberInfo)).Concat( lookup.OfType<MultipleMemberInfo>().SelectMany(mmi => mmi.Members) )); var result = new HashSet<OverloadResult>(OverloadResultComparer.Instance); // TODO: Include relevant type info on the parameter... result.UnionWith(lookup // Exclude constant values first time through .Where(av => av.MemberType != PythonMemberType.Constant) .SelectMany(av => av.Overloads ?? Enumerable.Empty<OverloadResult>()) ); if (!result.Any()) { result.UnionWith(lookup .Where(av => av.MemberType == PythonMemberType.Constant) .SelectMany(av => av.Overloads ?? Enumerable.Empty<OverloadResult>())); } return result; } } catch (Exception ex) { if (ex.IsCriticalException()) { throw; } Debug.Fail(ex.ToString()); return GetSignaturesError; } }
private Response IsMissingImport(AP.IsMissingImportRequest request) { var entry = _projectFiles[request.fileId] as IPythonProjectEntry; var analysis = entry.Analysis; if (analysis != null) { var location = new SourceLocation(request.index, request.line, request.column); var nameExpr = GetFirstNameExpression( analysis.GetAstFromText( request.text, location ).Body ); if (nameExpr != null && !IsImplicitlyDefinedName(nameExpr)) { var name = nameExpr.Name; var hasVariables = analysis.GetVariables(name, location).Any(IsDefinition); var hasValues = analysis.GetValues(name, location).Any(); // if we have type information or an assignment to the variable we won't offer // an import smart tag. if (!hasValues && !hasVariables) { return new AP.IsMissingImportResponse() { isMissing = true }; } } } return new AP.IsMissingImportResponse() { isMissing = false }; }
/// <summary> /// Gets the available names at the given location. This includes /// global variables and locals, but not built-in variables. /// </summary> /// <param name="location"> /// The location in the file where the available members should be /// looked up. /// </param> /// <remarks>TODO: Remove; this is only used for tests</remarks> /// <remarks>New in 2.2</remarks> internal IEnumerable<string> GetVariablesNoBuiltins(SourceLocation location) { var result = Enumerable.Empty<string>(); var chain = FindScope(location); foreach (var scope in chain.EnumerateFromGlobal) { if (scope.VisibleToChildren || scope == chain) { result = result.Concat(scope.GetAllMergedVariables().Select(val => val.Key)); } } return result.Distinct(); }
/// <summary> /// Gets the chain of scopes which are associated with the given position in the code. /// </summary> private InterpreterScope FindScope(SourceLocation location) { var res = FindScope(Scope, _unit.Tree, location); Debug.Assert(res != null, "Should never return null from FindScope"); return res; }
private static bool IsFirstLineOfFunction(InterpreterScope innerScope, InterpreterScope outerScope, SourceLocation location) { if (innerScope.OuterScope == outerScope && innerScope is FunctionScope) { var funcScope = (FunctionScope)innerScope; var def = funcScope.Function.FunctionDefinition; // TODO: Use indexes rather than lines to check location if (location.Line == def.GetStart(def.GlobalParent).Line) { return true; } } return false; }
/// <summary> /// Gets the available names at the given location. This includes /// built-in variables, global variables, and locals. /// </summary> /// <param name="location"> /// The location in the file where the available members should be /// looked up. /// </param> /// <remarks>New in 2.2</remarks> public IEnumerable<MemberResult> GetAllAvailableMembers(SourceLocation location, GetMemberOptions options = GetMemberOptions.IntersectMultipleResults) { var result = new Dictionary<string, IEnumerable<AnalysisValue>>(); // collect builtins if (!options.HasFlag(GetMemberOptions.ExcludeBuiltins)) { foreach (var variable in ProjectState.BuiltinModule.GetAllMembers(ProjectState._defaultContext)) { result[variable.Key] = new List<AnalysisValue>(variable.Value); } } // collect variables from user defined scopes var scope = FindScope(location); foreach (var s in scope.EnumerateTowardsGlobal) { foreach (var kvp in s.GetAllMergedVariables()) { // deliberately overwrite variables from outer scopes result[kvp.Key] = new List<AnalysisValue>(kvp.Value.TypesNoCopy); } } var res = MemberDictToResultList(GetPrivatePrefix(scope), options, result); if (options.Keywords()) { res = GetKeywordMembers(options, scope).Union(res); } return res; }
public void OnBeginStepOut(DkmThread thread) { // When we're stepping out while in Python code, there are two possibilities. Either the stack looks like this: // // PythonFrame1 // PythonFrame2 // // or else it looks like this: // // PythonFrame // [Native to Python transition] // NativeFrame // // In both cases, we use native breakpoints on the return address to catch the end of step-out operation. // For Python-to-native step-out, this is the only option. For Python-to-Python, it would seem that TraceFunc // can detect it via PyTrace_RETURN, but it doesn't actually know whether the return is to Python or to // native at the point where it's reported - and, in any case, we need to let PyEval_EvalFrameEx to return // before reporting the completion of that step-out (otherwise we will show the returning frame in call stack). // Find the destination for step-out by walking the call stack and finding either the first native frame // outside of Python and helper DLLs, or the second Python frame. var inspectionSession = DkmInspectionSession.Create(_process, null); var frameFormatOptions = new DkmFrameFormatOptions(DkmVariableInfoFlags.None, DkmFrameNameFormatOptions.None, DkmEvaluationFlags.None, 10000, 10); var stackContext = DkmStackContext.Create(inspectionSession, thread, DkmCallStackFilterOptions.None, frameFormatOptions, null, null); DkmStackFrame frame = null; for (int pyFrameCount = 0; pyFrameCount != 2; ) { DkmStackFrame[] frames = null; var workList = DkmWorkList.Create(null); stackContext.GetNextFrames(workList, 1, (result) => { frames = result.Frames; }); workList.Execute(); if (frames == null || frames.Length != 1) { return; } frame = frames[0]; var frameModuleInstance = frame.ModuleInstance; if (frameModuleInstance is DkmNativeModuleInstance && frameModuleInstance != _pyrtInfo.DLLs.Python && frameModuleInstance != _pyrtInfo.DLLs.DebuggerHelper && frameModuleInstance != _pyrtInfo.DLLs.CTypes) { break; } else if (frame.RuntimeInstance != null && frame.RuntimeInstance.Id.RuntimeType == Guids.PythonRuntimeTypeGuid) { ++pyFrameCount; } } var nativeAddr = frame.InstructionAddress as DkmNativeInstructionAddress; if (nativeAddr == null) { var customAddr = frame.InstructionAddress as DkmCustomInstructionAddress; if (customAddr == null) { return; } var loc = new SourceLocation(customAddr.AdditionalData, thread.Process); nativeAddr = loc.NativeAddress; if (nativeAddr == null) { return; } } var bp = DkmRuntimeInstructionBreakpoint.Create(Guids.PythonStepTargetSourceGuid, thread, nativeAddr, false, null); bp.Enable(); _stepOutTargetBreakpoints.Add(bp); }