public AstPythonFunction(PythonAst ast, IPythonModule declModule, IPythonType declType, FunctionDefinition def, string doc) {
DeclaringModule = declModule;
DeclaringType = declType;
Name = def.Name;
Documentation = doc;
foreach (var dec in (def.Decorators?.Decorators).MaybeEnumerate().OfType<NameExpression>()) {
if (dec.Name == "classmethod") {
IsClassMethod = true;
} else if (dec.Name == "staticmethod") {
IsStatic = true;
}
}
_overloads = new List<AstPythonFunctionOverload> {
new AstPythonFunctionOverload(Documentation, "", MakeParameters(ast, def), MakeReturns(def))
};
}
public override bool Walk(FunctionDefinition node) {
_names.Add(node.Name);
return base.Walk(node);
}
private void PushFunction(FunctionDefinition function) {
if (_functions == null) {
_functions = new Stack<FunctionDefinition>();
}
_functions.Push(function);
}
// Helpers for parsing lambda expressions.
// Usage
// FunctionDefinition f = ParseLambdaHelperStart(string);
// Expression expr = ParseXYZ();
// return ParseLambdaHelperEnd(f, expr);
private FunctionDefinition ParseLambdaHelperStart(out List<string> commaWhiteSpace, out bool ateTerminator) {
var start = GetStart();
Parameter[] parameters;
parameters = ParseVarArgsList(TokenKind.Colon, out commaWhiteSpace, out ateTerminator);
var mid = GetEnd();
FunctionDefinition func = new FunctionDefinition(null, parameters ?? new Parameter[0]); // new Parameter[0] for error handling of incomplete lambda
func.HeaderIndex = mid;
func.StartIndex = start;
// Push the lambda function on the stack so that it's available for any yield expressions to mark it as a generator.
PushFunction(func);
return func;
}
public ExtractMethodResult GetExtractionResult(ExtractMethodRequest info) {
bool isStaticMethod = false, isClassMethod = false;
var parameters = new List<Parameter>();
string selfParam = null;
if (info.TargetScope is ClassDefinition) {
var fromScope = _scopes[_scopes.Length - 1] as FunctionDefinition;
Debug.Assert(fromScope != null); // we don't allow extracting from classes, so we have to be coming from a function
if (fromScope != null) {
if (fromScope.Decorators != null) {
foreach (var decorator in fromScope.Decorators.Decorators) {
NameExpression name = decorator as NameExpression;
if (name != null) {
if (name.Name == "staticmethod") {
isStaticMethod = true;
} else if (name.Name == "classmethod") {
isClassMethod = true;
}
}
}
}
if (!isStaticMethod) {
if (fromScope.Parameters.Count > 0) {
selfParam = fromScope.Parameters[0].Name;
parameters.Add(new Parameter(selfParam, ParameterKind.Normal));
}
}
}
}
foreach (var param in info.Parameters) {
var newParam = new Parameter(param, ParameterKind.Normal);
if (parameters.Count > 0) {
newParam.AddPreceedingWhiteSpace(_ast, " ");
}
parameters.Add(newParam);
}
// include any non-closed over parameters as well...
foreach (var input in _inputVars) {
var variableScope = input.Scope;
var parentScope = info.TargetScope;
// are these variables a child of the target scope so we can close over them?
while (parentScope != null && parentScope != variableScope) {
parentScope = parentScope.Parent;
}
if (parentScope == null && input.Name != selfParam) {
// we can either close over or pass these in as parameters, add them to the list
var newParam = new Parameter(input.Name, ParameterKind.Normal);
if (parameters.Count > 0) {
newParam.AddPreceedingWhiteSpace(_ast, " ");
}
parameters.Add(newParam);
}
}
var body = _target.GetBody(_ast);
var isCoroutine = IsCoroutine(body);
// reset leading indentation to single newline + indentation, this
// strips out any proceeding comments which we don't extract
var leading = _newline + body.GetIndentationLevel(_ast);
body.SetLeadingWhiteSpace(_ast, leading);
if (_outputVars.Count > 0) {
// need to add a return statement
Expression retValue;
Expression[] names = new Expression[_outputVars.Count];
int outputIndex = 0;
foreach (var name in _outputVars) {
var nameExpr = new NameExpression(name.Name);
nameExpr.AddPreceedingWhiteSpace(_ast, " ");
names[outputIndex++] = nameExpr;
}
var tuple = new TupleExpression(false, names);
tuple.RoundTripHasNoParenthesis(_ast);
retValue = tuple;
var retStmt = new ReturnStatement(retValue);
retStmt.SetLeadingWhiteSpace(_ast, leading);
body = new SuiteStatement(
new Statement[] {
body,
retStmt
}
);
} else {
// we need a SuiteStatement to give us our colon
body = new SuiteStatement(new Statement[] { body });
}
DecoratorStatement decorators = null;
if (isStaticMethod) {
decorators = new DecoratorStatement(new[] { new NameExpression("staticmethod") });
} else if (isClassMethod) {
decorators = new DecoratorStatement(new[] { new NameExpression("classmethod") });
}
var res = new FunctionDefinition(new NameExpression(info.Name), parameters.ToArray(), body, decorators);
res.IsCoroutine = isCoroutine;
StringBuilder newCall = new StringBuilder();
newCall.Append(_target.IndentationLevel);
var method = res.ToCodeString(_ast);
// fix up indentation...
for (int curScope = 0; curScope < _scopes.Length; curScope++) {
if (_scopes[curScope] == info.TargetScope) {
// this is our target indentation level.
var indentationLevel = _scopes[curScope].Body.GetIndentationLevel(_ast);
var lines = method.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None);
int minWhiteSpace = Int32.MaxValue;
for (int curLine = decorators == null ? 1 : 2; curLine < lines.Length; curLine++) {
var line = lines[curLine];
for (int i = 0; i < line.Length; i++) {
if (!Char.IsWhiteSpace(line[i])) {
minWhiteSpace = Math.Min(minWhiteSpace, i);
break;
}
}
}
StringBuilder newLines = new StringBuilder();
newLines.Append(indentationLevel);
newLines.Append(lines[0]);
if (decorators != null) {
newLines.Append(_newline);
newLines.Append(indentationLevel);
newLines.Append(lines[1]);
}
// don't include a bunch of blank lines...
int endLine = lines.Length - 1;
for (; endLine >= 0 && String.IsNullOrWhiteSpace(lines[endLine]); endLine--) {
}
newLines.Append(_newline);
for (int curLine = decorators == null ? 1 : 2; curLine <= endLine; curLine++) {
var line = lines[curLine];
newLines.Append(indentationLevel);
if (_insertTabs) {
newLines.Append('\t');
} else {
newLines.Append(' ', _indentSize);
}
if (line.Length > minWhiteSpace) {
newLines.Append(line, minWhiteSpace, line.Length - minWhiteSpace);
}
newLines.Append(_newline);
}
newLines.Append(_newline);
method = newLines.ToString();
break;
}
}
string comma;
if (_outputVars.Count > 0) {
comma = "";
foreach (var outputVar in _outputVars) {
newCall.Append(comma);
newCall.Append(outputVar.Name);
comma = ", ";
}
newCall.Append(" = ");
} else if (_target.ContainsReturn) {
newCall.Append("return ");
}
if (isCoroutine) {
newCall.Append("await ");
}
if (info.TargetScope is ClassDefinition) {
var fromScope = _scopes[_scopes.Length - 1] as FunctionDefinition;
Debug.Assert(fromScope != null); // we don't allow extracting from classes, so we have to be coming from a function
if (isStaticMethod) {
newCall.Append(info.TargetScope.Name);
newCall.Append('.');
} else if (fromScope != null && fromScope.Parameters.Count > 0) {
newCall.Append(fromScope.Parameters[0].Name);
newCall.Append('.');
}
}
newCall.Append(info.Name);
newCall.Append('(');
comma = "";
foreach (var param in parameters) {
if (param.Name != selfParam) {
newCall.Append(comma);
newCall.Append(param.Name);
comma = ", ";
}
}
newCall.Append(')');
return new ExtractMethodResult(
method,
newCall.ToString()
);
}
public override bool Walk(FunctionDefinition node) {
if (node.IsCoroutine) {
AddSpan(Tuple.Create("", new Span(node.StartIndex, 5)), PredefinedClassificationTypeNames.Keyword);
}
Debug.Assert(_head != null);
_head.Functions.Add(node.NameExpression.Name);
node.NameExpression.Walk(this);
BeginScope();
return base.Walk(node);
}
// FunctionDefinition
public override bool Walk(FunctionDefinition node) {
node._nameVariable = _globalScope.EnsureGlobalVariable("__name__");
// Name is defined in the enclosing context
if (!node.IsLambda) {
node.Variable = DefineName(node.Name);
node.AddVariableReference(_globalScope, _bindRefs, Reference(node.Name));
}
// process the default arg values and annotations in the outer
// context
foreach (Parameter p in node.Parameters) {
if (p.DefaultValue != null) {
p.DefaultValue.Walk(this);
}
if (p.Annotation != null) {
p.Annotation.Walk(this);
}
}
// process the decorators in the outer context
if (node.Decorators != null) {
foreach (Expression dec in node.Decorators.Decorators) {
if (dec != null) {
dec.Walk(this);
}
}
}
// process the return annotation in the outer context
if (node.ReturnAnnotation != null) {
node.ReturnAnnotation.Walk(this);
}
PushScope(node);
foreach (Parameter p in node.Parameters) {
p.Walk(_parameter);
}
if (node.Body != null) {
node.Body.Walk(this);
}
return false;
}
public override bool Walk(FunctionDefinition node) {
var function = AddFunction(node, _curUnit);
if (function != null) {
_analysisStack.Push(_curUnit);
_curUnit = function.AnalysisUnit;
Debug.Assert(_scope.EnumerateTowardsGlobal.Contains(function.AnalysisUnit.Scope.OuterScope));
_scope = function.AnalysisUnit.Scope;
return true;
}
return false;
}
private static IEnumerable<IPythonType> MakeReturns(FunctionDefinition def) {
yield break;
}
private static IEnumerable<IParameterInfo> MakeParameters(PythonAst ast, FunctionDefinition def) {
foreach (var p in def.Parameters) {
yield return new AstPythonParameterInfo(ast, p);
}
}
internal void AddOverload(PythonAst ast, FunctionDefinition def) {
_overloads.Add(new AstPythonFunctionOverload(def.Documentation, "", MakeParameters(ast, def), MakeReturns(def)));
}
public LambdaExpression(FunctionDefinition function) {
_function = function;
}
public override void PostWalk(FunctionDefinition node) {
_scopes.Add(node);
_allWrites.Add(node.GetVariableReference(_root));
_allWrittenVariables.Add(node.Variable);
base.PostWalk(node);
}
public override void PostWalk(FunctionDefinition node) {
_scopes.Add(node);
base.PostWalk(node);
}
public override bool Walk(FunctionDefinition node) {
if (!node.IsLambda) {
_define.WalkName(node.NameExpression, node.GetVariableReference(_root));
}
bool oldInLoop = _inLoop;
_inLoop = false;
var res = base.Walk(node);
_inLoop = oldInLoop;
return res;
}
// FunctionDefinition
public override void PostWalk(FunctionDefinition node)
{
Debug.Assert(_currentScope == node);
PopScope();
}
internal static FunctionInfo AddFunction(FunctionDefinition node, AnalysisUnit outerUnit, InterpreterScope prevScope) {
InterpreterScope scope;
if (!prevScope.TryGetNodeScope(node, out scope)) {
if (node.Body == null || node.Name == null) {
return null;
}
var func = new FunctionInfo(node, outerUnit, prevScope);
var unit = func.AnalysisUnit;
scope = unit.Scope;
prevScope.Children.Add(scope);
prevScope.AddNodeScope(node, scope);
if (!node.IsLambda && node.Name != "<genexpr>") {
// lambdas don't have their names published
var funcVar = prevScope.AddLocatedVariable(node.Name, node.NameExpression, unit);
// Decorated functions don't have their type set yet
if (node.Decorators == null) {
funcVar.AddTypes(unit, func.SelfSet);
}
}
unit.Enqueue();
}
return scope.AnalysisValue as FunctionInfo;
}
public override void PostWalk(FunctionDefinition node) {
int start = node.GetStart(_ast).Line;
int end = node.Body.GetEnd(_ast).Line + 1;
if (_lineNumber < start || _lineNumber >= end) {
return;
}
string funcName = node.Name;
if (_name.Length == 0 && funcName != _expectedFuncName) {
// The innermost function name must match the one that we've got from the code object.
// If it doesn't, the source code that we're parsing is out of sync with the running program,
// and cannot be used to compute the fully qualified name.
throw new InvalidDataException();
}
for (var classDef = node.Parent as ClassDefinition; classDef != null; classDef = classDef.Parent as ClassDefinition) {
funcName = classDef.Name + "." + funcName;
}
if (_name.Length != 0) {
_name.Append(" in ");
}
_name.Append(funcName);
}
public override bool Walk(FunctionDefinition node) {
var newScope = WalkScope(node);
CurScope.Children.Add(newScope);
return false;
}
private void GetFunctionDescription(FunctionDefinition def, Action<string, VSOBDESCRIPTIONSECTION, IVsNavInfo> addDescription) {
addDescription(ScopeNode.Name, VSOBDESCRIPTIONSECTION.OBDS_NAME, null);
addDescription("(", VSOBDESCRIPTIONSECTION.OBDS_MISC, null);
for (int i = 0; i < def.Parameters.Count; i++) {
if (i != 0) {
addDescription(", ", VSOBDESCRIPTIONSECTION.OBDS_MISC, null);
}
var curParam = def.Parameters[i];
string name = curParam.Name;
if (curParam.IsDictionary) {
name = "**" + name;
} else if (curParam.IsList) {
name = "*" + curParam.Name;
}
if (curParam.DefaultValue != null) {
// TODO: Support all possible expressions for default values, we should
// probably have a PythonAst walker for expressions or we should add ToCodeString()
// onto Python ASTs so they can round trip
ConstantExpression defaultValue = curParam.DefaultValue as ConstantExpression;
if (defaultValue != null) {
// FIXME: Use python repr
name = name + " = " + defaultValue.GetConstantRepr(def.GlobalParent.LanguageVersion);
}
}
addDescription(name, VSOBDESCRIPTIONSECTION.OBDS_PARAM, null);
}
addDescription(")\n", VSOBDESCRIPTIONSECTION.OBDS_MISC, null);
}
// FunctionDefinition
public override void PostWalk(FunctionDefinition node) {
Debug.Assert(_currentScope == node);
PopScope();
}
public FunctionScopeNode(FunctionDefinition func) {
_func = func;
}
public override void PostWalk(FunctionDefinition node) {
EndScope(false);
Debug.Assert(_head != null);
base.PostWalk(node);
}
public override bool Walk(FunctionDefinition node) {
var existing = GetInScope(node.Name) as AstPythonFunction;
if (existing == null) {
var m = _scope.Peek();
if (m != null) {
m[node.Name] = new AstPythonFunction(_ast, _module, CurrentClass, node, GetDoc(node.Body as SuiteStatement));
}
} else {
existing.AddOverload(_ast, node);
}
// Do not recurse into functions
return false;
}
// funcdef: [decorators] 'def' NAME parameters ':' suite
// parameters: '(' [varargslist] ')'
// this gets called with "def" as the look-ahead
private FunctionDefinition ParseFuncDef(bool isCoroutine) {
string preWhitespace = null, afterAsyncWhitespace = null;
var start = isCoroutine ? GetStart() : 0;
if (isCoroutine) {
preWhitespace = _tokenWhiteSpace;
}
Eat(TokenKind.KeywordDef);
if (isCoroutine) {
afterAsyncWhitespace = _tokenWhiteSpace;
} else {
preWhitespace = _tokenWhiteSpace;
start = GetStart();
}
var name = ReadName();
var nameExpr = MakeName(name);
nameExpr.SetLoc(GetStart(), GetEnd());
string nameWhiteSpace = _tokenWhiteSpace;
bool ateLeftParen = name.HasName && Eat(TokenKind.LeftParenthesis);
string parenWhiteSpace = _tokenWhiteSpace;
var lStart = GetStart();
var lEnd = GetEnd();
int grouping = _tokenizer.GroupingLevel;
List<string> commaWhiteSpace = null;
bool ateTerminator = false;
Parameter[] parameters = ateLeftParen ? ParseVarArgsList(TokenKind.RightParenthesis, out commaWhiteSpace, out ateTerminator, true) : null;
string closeParenWhiteSpace = _tokenWhiteSpace;
FunctionDefinition ret;
if (parameters == null) {
// error in parameters
ret = new FunctionDefinition(nameExpr, new Parameter[0]);
ret.IsCoroutine = isCoroutine;
if (_verbatim) {
AddVerbatimName(name, ret);
AddPreceedingWhiteSpace(ret, preWhitespace);
if (afterAsyncWhitespace != null) {
GetNodeAttributes(ret)[FunctionDefinition.WhitespaceAfterAsync] = afterAsyncWhitespace;
}
AddSecondPreceedingWhiteSpace(ret, nameWhiteSpace);
AddThirdPreceedingWhiteSpace(ret, parenWhiteSpace);
AddFourthPreceedingWhiteSpace(ret, closeParenWhiteSpace);
if (!ateTerminator) {
AddErrorMissingCloseGrouping(ret);
}
if (!ateLeftParen) {
AddErrorIsIncompleteNode(ret);
}
}
ret.SetLoc(start, lEnd);
return ret;
}
string arrowWhiteSpace = null;
Expression returnAnnotation = null;
if (MaybeEat(TokenKind.Arrow)) {
arrowWhiteSpace = _tokenWhiteSpace;
returnAnnotation = ParseExpression();
}
var rStart = GetStart();
var rEnd = GetEnd();
ret = new FunctionDefinition(nameExpr, parameters);
AddVerbatimName(name, ret);
PushFunction(ret);
// set IsCoroutine before parsing the body to enable use of 'await'
ret.IsCoroutine = isCoroutine;
Statement body = ParseClassOrFuncBody();
FunctionDefinition ret2 = PopFunction();
System.Diagnostics.Debug.Assert(ret == ret2);
ret.SetBody(body);
ret.ReturnAnnotation = returnAnnotation;
ret.HeaderIndex = rEnd;
if (_verbatim) {
AddPreceedingWhiteSpace(ret, preWhitespace);
if (afterAsyncWhitespace != null) {
GetNodeAttributes(ret)[FunctionDefinition.WhitespaceAfterAsync] = afterAsyncWhitespace;
}
AddSecondPreceedingWhiteSpace(ret, nameWhiteSpace);
AddThirdPreceedingWhiteSpace(ret, parenWhiteSpace);
AddFourthPreceedingWhiteSpace(ret, closeParenWhiteSpace);
AddListWhiteSpace(ret, commaWhiteSpace.ToArray());
if (arrowWhiteSpace != null) {
AddFifthPreceedingWhiteSpace(ret, arrowWhiteSpace);
}
if (!ateTerminator) {
AddErrorMissingCloseGrouping(ret);
}
}
ret.SetLoc(start, body.EndIndex);
return ret;
}
public override void PostWalk(FunctionDefinition node) {
_scopes.RemoveAt(_scopes.Count - 1);
base.PostWalk(node);
}
private Expression ParseLambdaHelperEnd(FunctionDefinition func, Expression expr, string whitespace, string colonWhiteSpace, List<string> commaWhiteSpace, bool ateTerminator) {
// Pep 342 in Python 2.5 allows Yield Expressions, which can occur inside a Lambda body.
// In this case, the lambda is a generator and will yield it's final result instead of just return it.
Statement body;
if (func.IsGenerator) {
YieldExpression y = new YieldExpression(expr);
y.SetLoc(expr.IndexSpan);
body = new ExpressionStatement(y);
} else {
body = new ReturnStatement(expr);
}
body.SetLoc(expr.StartIndex, expr.EndIndex);
FunctionDefinition func2 = PopFunction();
System.Diagnostics.Debug.Assert(func == func2);
func.SetBody(body);
func.EndIndex = GetEnd();
LambdaExpression ret = new LambdaExpression(func);
func.SetLoc(func.IndexSpan);
ret.SetLoc(func.IndexSpan);
if (_verbatim) {
AddPreceedingWhiteSpace(ret, whitespace);
AddSecondPreceedingWhiteSpace(ret, colonWhiteSpace);
AddListWhiteSpace(ret, commaWhiteSpace.ToArray());
if (!ateTerminator) {
AddErrorIsIncompleteNode(ret);
}
}
return ret;
}
public override void PostWalk(FunctionDefinition node) {
if (node.Body != null && node.Name != null) {
Debug.Assert(_scope.EnumerateTowardsGlobal.Contains(_curUnit.Scope.OuterScope));
_scope = _curUnit.Scope.OuterScope;
_curUnit = _analysisStack.Pop();
Debug.Assert(_scope.EnumerateTowardsGlobal.Contains(_curUnit.Scope));
}
}
private static ImageListKind GetImageListKind(FunctionDefinition funcDef) {
ImageListKind imageKind = ImageListKind.Method;
if (funcDef.Decorators != null && funcDef.Decorators.Decorators.Count == 1) {
foreach (var decorator in funcDef.Decorators.Decorators) {
NameExpression nameExpr = decorator as NameExpression;
if (nameExpr != null) {
if (nameExpr.Name == "property") {
imageKind = ImageListKind.Property;
break;
} else if (nameExpr.Name == "staticmethod") {
imageKind = ImageListKind.StaticMethod;
break;
} else if (nameExpr.Name == "classmethod") {
imageKind = ImageListKind.ClassMethod;
break;
}
}
}
}
return imageKind;
}
internal FunctionInfo AddFunction(FunctionDefinition node, AnalysisUnit outerUnit) {
return AddFunction(node, outerUnit, _scope);
}
public override bool Walk(FunctionDefinition node) {
CommitByDefault = true;
if (node.Parameters != null) {
CanComplete = false;
foreach (var p in node.Parameters) {
p.Walk(this);
}
}
if (node.Decorators != null) {
node.Decorators.Walk(this);
}
if (node.ReturnAnnotation != null) {
CanComplete = true;
node.ReturnAnnotation.Walk(this);
}
if (node.Body != null && !(node.Body is ErrorStatement)) {
CanComplete = node.IsLambda;
node.Body.Walk(this);
}
return false;
}
public LambdaExpression(FunctionDefinition function)
{
_function = function;
}