public override void PostWalk(FunctionDefinition node)
{
if (node.Body != null && node.Name != null) {
Debug.Assert(_scope.Node == node);
Debug.Assert(_scope.OuterScope.Node != node);
_scope = _scope.OuterScope;
_curUnit = _analysisStack.Pop();
Debug.Assert(_scope.EnumerateTowardsGlobal.Contains(_curUnit.Scope));
}
}
internal FunctionInfo(FunctionDefinition node, AnalysisUnit declUnit, InterpreterScope declScope)
{
_projectEntry = declUnit.ProjectEntry;
_functionDefinition = node;
_declVersion = declUnit.ProjectEntry.AnalysisVersion;
if (Name == "__new__") {
IsClassMethod = true;
}
_analysisUnit = new FunctionAnalysisUnit(this, declUnit, declScope);
}
public FunctionScopeNode(FunctionDefinition func)
{
_func = func;
}
public LambdaExpression(FunctionDefinition function)
{
_function = function;
}
public override bool Walk(FunctionDefinition node)
{
_scopes.Add(node);
return base.Walk(node);
}
// FuncDef
public override bool Walk(FunctionDefinition node)
{
if (node == _scope) {
// the function body is being analyzed, go deep:
foreach (Parameter p in node.Parameters) {
p.Walk(_fdef);
}
return true;
} else {
// analyze the function definition itself (it is visited while analyzing parent scope):
Define(node.Name);
foreach (Parameter p in node.Parameters) {
if (p.DefaultValue != null) {
p.DefaultValue.Walk(this);
}
}
return false;
}
}
public override bool Walk(FunctionDefinition node)
{
if (!node.IsLambda) {
_define.WalkName(node.NameExpression, node.GetVariableReference(_root));
}
bool oldInLoop = _inLoop;
_inLoop = false;
var res = base.Walk(node);
_inLoop = oldInLoop;
return res;
}
public override void PostWalk(FunctionDefinition node)
{
_scopes.RemoveAt(_scopes.Count - 1);
base.PostWalk(node);
}
private Expression ParseLambdaHelperEnd(FunctionDefinition func, Expression expr, string whitespace, string colonWhiteSpace, List<string> commaWhiteSpace, bool ateTerminator)
{
// Pep 342 in J 2.5 allows Yield Expressions, which can occur inside a Lambda body.
// In this case, the lambda is a generator and will yield it's final result instead of just return it.
Statement body;
if (func.IsGenerator) {
YieldExpression y = new YieldExpression(expr);
y.SetLoc(expr.IndexSpan);
body = new ExpressionStatement(y);
} else {
body = new ReturnStatement(expr);
}
body.SetLoc(expr.StartIndex, expr.EndIndex);
FunctionDefinition func2 = PopFunction();
System.Diagnostics.Debug.Assert(func == func2);
func.SetBody(body);
func.EndIndex = GetEnd();
LambdaExpression ret = new LambdaExpression(func);
func.SetLoc(func.IndexSpan);
ret.SetLoc(func.IndexSpan);
if (_verbatim) {
AddPreceedingWhiteSpace(ret, whitespace);
AddSecondPreceedingWhiteSpace(ret, colonWhiteSpace);
AddListWhiteSpace(ret, commaWhiteSpace.ToArray());
if (!ateTerminator) {
AddErrorIsIncompleteNode(ret);
}
}
return ret;
}
private void PushFunction(FunctionDefinition function)
{
if (_functions == null) {
_functions = new Stack<FunctionDefinition>();
}
_functions.Push(function);
}
internal FunctionInfo AddFunction(FunctionDefinition node, AnalysisUnit outerUnit)
{
return AddFunction(node, outerUnit, _scope);
}
private static ImageListKind GetImageListKind(FunctionDefinition funcDef)
{
ImageListKind imageKind = ImageListKind.Method;
if (funcDef.Decorators != null && funcDef.Decorators.Decorators.Count == 1) {
foreach (var decorator in funcDef.Decorators.Decorators) {
NameExpression nameExpr = decorator as NameExpression;
if (nameExpr != null) {
if (nameExpr.Name == "property") {
imageKind = ImageListKind.Property;
break;
} else if (nameExpr.Name == "staticmethod") {
imageKind = ImageListKind.StaticMethod;
break;
} else if (nameExpr.Name == "classmethod") {
imageKind = ImageListKind.ClassMethod;
break;
}
}
}
}
return imageKind;
}
internal static FunctionInfo AddFunction(FunctionDefinition node, AnalysisUnit outerUnit, InterpreterScope prevScope)
{
InterpreterScope scope;
if (!prevScope.TryGetNodeScope(node, out scope)) {
if (node.Body == null || node.Name == null) {
return null;
}
var func = new FunctionInfo(node, outerUnit, prevScope);
var unit = func.AnalysisUnit;
scope = unit.Scope;
prevScope.Children.Add(scope);
prevScope.AddNodeScope(node, scope);
if (!node.IsLambda && node.Name != "<genexpr>") {
// lambdas don't have their names published
var funcVar = prevScope.AddLocatedVariable(node.Name, node.NameExpression, unit);
// Decorated functions don't have their type set yet
if (node.Decorators == null) {
funcVar.AddTypes(unit, func.SelfSet);
}
}
unit.Enqueue();
}
return scope.Namespace as FunctionInfo;
}
public override bool Walk(FunctionDefinition node)
{
var function = AddFunction(node, _curUnit);
if (function != null) {
_analysisStack.Push(_curUnit);
_curUnit = function.AnalysisUnit;
Debug.Assert(_scope.EnumerateTowardsGlobal.Contains(function.AnalysisUnit.Scope.OuterScope));
_scope = function.AnalysisUnit.Scope;
return true;
}
return false;
}
// FunctionDefinition
public override bool Walk(FunctionDefinition node)
{
return ShouldWalkWorker(node);
}
public override void PostWalk(FunctionDefinition node)
{
PostWalkWorker(node);
}
public override bool Walk(FunctionDefinition node)
{
return false;
}
// funcdef: [decorators] 'def' NAME parameters ':' suite
// parameters: '(' [varargslist] ')'
// this gets called with "def" as the look-ahead
private FunctionDefinition ParseFuncDef()
{
Eat(TokenKind.KeywordDef);
string defWhitespace = _tokenWhiteSpace;
var start = GetStart();
var name = ReadName();
var nameExpr = MakeName(name);
nameExpr.SetLoc(GetStart(), GetEnd());
string nameWhiteSpace = _tokenWhiteSpace;
bool ateLeftParen = name.HasName && Eat(TokenKind.LeftParenthesis);
string parenWhiteSpace = _tokenWhiteSpace;
var lStart = GetStart();
var lEnd = GetEnd();
int grouping = _tokenizer.GroupingLevel;
List<string> commaWhiteSpace = null;
bool ateTerminator = false;
Parameter[] parameters = ateLeftParen ? ParseVarArgsList(TokenKind.RightParenthesis, out commaWhiteSpace, out ateTerminator, true) : null;
string closeParenWhiteSpace = _tokenWhiteSpace;
FunctionDefinition ret;
if (parameters == null) {
// error in parameters
ret = new FunctionDefinition(nameExpr, new Parameter[0]);
if (_verbatim) {
AddVerbatimName(name, ret);
AddPreceedingWhiteSpace(ret, defWhitespace);
AddSecondPreceedingWhiteSpace(ret, nameWhiteSpace);
AddThirdPreceedingWhiteSpace(ret, parenWhiteSpace);
AddFourthPreceedingWhiteSpace(ret, closeParenWhiteSpace);
if (!ateTerminator) {
AddErrorMissingCloseGrouping(ret);
}
if (!ateLeftParen) {
AddErrorIsIncompleteNode(ret);
}
}
ret.SetLoc(start, lEnd);
return ret;
}
var rStart = GetStart();
var rEnd = GetEnd();
ret = new FunctionDefinition(nameExpr, parameters);
AddVerbatimName(name, ret);
PushFunction(ret);
string arrowWhiteSpace = null;
if (MaybeEat(TokenKind.Arrow)) {
arrowWhiteSpace = _tokenWhiteSpace;
ret.ReturnAnnotation = ParseExpression();
}
Statement body = ParseClassOrFuncBody();
FunctionDefinition ret2 = PopFunction();
System.Diagnostics.Debug.Assert(ret == ret2);
ret.SetBody(body);
ret.HeaderIndex = rEnd;
if (_verbatim) {
AddPreceedingWhiteSpace(ret, defWhitespace);
AddSecondPreceedingWhiteSpace(ret, nameWhiteSpace);
AddThirdPreceedingWhiteSpace(ret, parenWhiteSpace);
AddFourthPreceedingWhiteSpace(ret, closeParenWhiteSpace);
AddListWhiteSpace(ret, commaWhiteSpace.ToArray());
if (arrowWhiteSpace != null) {
AddFifthPreceedingWhiteSpace(ret, arrowWhiteSpace);
}
if (!ateTerminator) {
AddErrorMissingCloseGrouping(ret);
}
}
ret.SetLoc(start, body.EndIndex);
return ret;
}
public ExtractMethodResult GetExtractionResult(ExtractMethodRequest info)
{
bool isStaticMethod = false, isClassMethod = false;
var parameters = new List<Parameter>();
string selfParam = null;
if (info.TargetScope is ClassDefinition) {
var fromScope = _scopes[_scopes.Length - 1] as FunctionDefinition;
Debug.Assert(fromScope != null); // we don't allow extracting from classes, so we have to be coming from a function
if (fromScope.Decorators != null) {
foreach (var decorator in fromScope.Decorators.Decorators) {
NameExpression name = decorator as NameExpression;
if (name != null) {
if (name.Name == "staticmethod") {
isStaticMethod = true;
} else if (name.Name == "classmethod") {
isClassMethod = true;
}
}
}
}
if (!isStaticMethod) {
if (fromScope.Parameters.Count > 0) {
selfParam = fromScope.Parameters[0].Name;
parameters.Add(new Parameter(selfParam, ParameterKind.Normal));
}
}
}
foreach (var param in info.Parameters) {
var newParam = new Parameter(param, ParameterKind.Normal);
if (parameters.Count > 0) {
newParam.AddPreceedingWhiteSpace(_ast, " ");
}
parameters.Add(newParam);
}
// include any non-closed over parameters as well...
foreach (var input in _inputVars) {
var variableScope = input.Scope;
var parentScope = info.TargetScope;
// are these variables a child of the target scope so we can close over them?
while (parentScope != null && parentScope != variableScope) {
parentScope = parentScope.Parent;
}
if (parentScope == null && input.Name != selfParam) {
// we can either close over or pass these in as parameters, add them to the list
var newParam = new Parameter(input.Name, ParameterKind.Normal);
if (parameters.Count > 0) {
newParam.AddPreceedingWhiteSpace(_ast, " ");
}
parameters.Add(newParam);
}
}
var body = _target.GetBody(_ast);
if (_outputVars.Count > 0) {
// need to add a return statement
Expression retValue;
Expression[] names = new Expression[_outputVars.Count];
int outputIndex = 0;
foreach (var name in _outputVars) {
var nameExpr = new NameExpression(name.Name);
nameExpr.AddPreceedingWhiteSpace(_ast, " ");
names[outputIndex++] = nameExpr;
}
var tuple = new TupleExpression(false, names);
tuple.RoundTripHasNoParenthesis(_ast);
retValue = tuple;
var retStmt = new ReturnStatement(retValue);
if (body is SuiteStatement) {
SuiteStatement suite = (SuiteStatement)body;
Node.CopyLeadingWhiteSpace(_ast, suite.Statements[0], retStmt);
Node.CopyTrailingNewLine(_ast, suite.Statements[0], suite.Statements[suite.Statements.Count - 1]);
Statement[] statements = new Statement[suite.Statements.Count + 1];
for (int i = 0; i < suite.Statements.Count; i++) {
statements[i] = suite.Statements[i];
}
statements[statements.Length - 1] = retStmt;
body = new SuiteStatement(statements);
} else {
Node.CopyLeadingWhiteSpace(_ast, body, retStmt);
body = new SuiteStatement(
new Statement[] {
body,
retStmt
}
);
}
}
DecoratorStatement decorators = null;
if (isStaticMethod) {
decorators = new DecoratorStatement(new[] { new NameExpression("staticmethod") });
} else if (isClassMethod) {
decorators = new DecoratorStatement(new[] { new NameExpression("classmethod") });
}
var res = new FunctionDefinition(new NameExpression(info.Name), parameters.ToArray(), body, decorators);
StringBuilder newCall = new StringBuilder();
newCall.Append(_target.IndentationLevel);
var method = res.ToCodeString(_ast);
// fix up indentation...
for (int curScope = 0; curScope < _scopes.Length; curScope++) {
if (_scopes[curScope] == info.TargetScope) {
// this is our target indentation level.
var indentationLevel = _scopes[curScope].Body.GetIndentationLevel(_ast);
var lines = method.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None);
int minWhiteSpace = Int32.MaxValue;
for (int curLine = decorators == null ? 1 : 2; curLine < lines.Length; curLine++) {
var line = lines[curLine];
for (int i = 0; i < line.Length; i++) {
if (!Char.IsWhiteSpace(line[i])) {
minWhiteSpace = Math.Min(minWhiteSpace, i);
break;
}
}
}
StringBuilder newLines = new StringBuilder();
newLines.Append(indentationLevel);
newLines.Append(lines[0]);
if (decorators != null) {
newLines.Append("\r\n");
newLines.Append(indentationLevel);
newLines.Append(lines[1]);
}
// don't include a bunch of blank lines...
int endLine = lines.Length - 1;
for (; endLine >= 0 && String.IsNullOrWhiteSpace(lines[endLine]); endLine--) {
}
newLines.Append("\r\n");
for (int curLine = decorators == null ? 1 : 2; curLine <= endLine; curLine++) {
var line = lines[curLine];
newLines.Append(indentationLevel);
if (_insertTabs) {
newLines.Append('\t');
} else {
newLines.Append(' ', _indentSize);
}
if (line.Length > minWhiteSpace) {
newLines.Append(line, minWhiteSpace, line.Length - minWhiteSpace);
}
newLines.Append("\r\n");
}
newLines.Append("\r\n");
method = newLines.ToString();
break;
}
}
string comma;
if (_outputVars.Count > 0) {
comma = "";
foreach (var outputVar in _outputVars) {
newCall.Append(comma);
newCall.Append(outputVar.Name);
comma = ", ";
}
newCall.Append(" = ");
} else if (_target.ContainsReturn) {
newCall.Append("return ");
}
if (info.TargetScope is ClassDefinition) {
var fromScope = _scopes[_scopes.Length - 1] as FunctionDefinition;
Debug.Assert(fromScope != null); // we don't allow extracting from classes, so we have to be coming from a function
if (isStaticMethod) {
newCall.Append(info.TargetScope.Name);
newCall.Append('.');
} else if (fromScope.Parameters.Count > 0) {
newCall.Append(fromScope.Parameters[0].Name);
newCall.Append('.');
}
}
newCall.Append(info.Name);
newCall.Append('(');
comma = "";
foreach (var param in parameters) {
if (param.Name != selfParam) {
newCall.Append(comma);
newCall.Append(param.Name);
comma = ", ";
}
}
newCall.Append(')');
return new ExtractMethodResult(
method,
newCall.ToString()
);
}
// Helpers for parsing lambda expressions.
// Usage
// FunctionDefinition f = ParseLambdaHelperStart(string);
// Expression expr = ParseXYZ();
// return ParseLambdaHelperEnd(f, expr);
private FunctionDefinition ParseLambdaHelperStart(out List<string> commaWhiteSpace, out bool ateTerminator)
{
var start = GetStart();
Parameter[] parameters;
parameters = ParseVarArgsList(TokenKind.Colon, out commaWhiteSpace, out ateTerminator);
var mid = GetEnd();
FunctionDefinition func = new FunctionDefinition(null, parameters ?? new Parameter[0]); // new Parameter[0] for error handling of incomplete lambda
func.HeaderIndex = mid;
func.StartIndex = start;
// Push the lambda function on the stack so that it's available for any yield expressions to mark it as a generator.
PushFunction(func);
return func;
}
public static ArgumentSet FromArgs(FunctionDefinition node, AnalysisUnit unit, INamespaceSet[] args, NameExpression[] keywordArgs)
{
// TODO: Warn when a keyword argument is provided and it maps to
// something which is also a positional argument:
// def f(a, b, c):
// print a, b, c
//
// f(2, 3, a=42)
int kwArgsOffset = args.Length - keywordArgs.Length;
var seqArgs = NamespaceSet.EmptyUnion;
var dictArgs = NamespaceSet.EmptyUnion;
int listArgsIndex = -1;
int dictArgsIndex = -1;
int argCount = node.Parameters.Count;
var newArgs = new INamespaceSet[argCount + 2];
for (int i = 0; i < node.Parameters.Count; ++i) {
if (node.Parameters[i].Kind == ParameterKind.List) {
listArgsIndex = i;
} else if (node.Parameters[i].Kind == ParameterKind.Dictionary) {
dictArgsIndex = i;
}
newArgs[i] = NamespaceSet.Empty;
}
for (int i = 0; i < kwArgsOffset; ++i) {
if (i < argCount && (listArgsIndex < 0 || i < listArgsIndex)) {
newArgs[i] = newArgs[i].Union(args[i]);
} else if (listArgsIndex >= 0) {
foreach (var ns in args[i]) {
var sseq = ns as StarArgsSequenceInfo;
if (sseq != null && i < node.Parameters.Count && sseq._node == node.Parameters[i]) {
seqArgs = seqArgs.Add(unit.ProjectState._tupleType.Instance);
} else {
seqArgs = seqArgs.Add(ns);
}
}
} else {
// TODO: Warn about extra parameters
}
}
for (int i = kwArgsOffset; i < args.Length; ++i) {
if (i - kwArgsOffset >= keywordArgs.Length || keywordArgs[i - kwArgsOffset] == null) {
continue;
}
var name = keywordArgs[i - kwArgsOffset].Name;
if (name.Equals("*", StringComparison.Ordinal)) {
if (listArgsIndex >= 0) {
foreach (var ns in args[i]) {
var sseq = ns as StarArgsSequenceInfo;
if (sseq != null) {
seqArgs = seqArgs.Union(sseq.IndexTypes.SelectMany(def => def.TypesNoCopy));
} else {
newArgs[listArgsIndex] = newArgs[listArgsIndex].Add(ns);
}
}
}
} else if (name.Equals("**", StringComparison.Ordinal)) {
if (dictArgsIndex >= 0) {
foreach (var ns in args[i]) {
var sdict = ns as StarArgsDictionaryInfo;
if (sdict != null) {
dictArgs = dictArgs.Union(sdict._keysAndValues.AllValueTypes);
} else {
newArgs[dictArgsIndex] = newArgs[dictArgsIndex].Add(ns);
}
}
}
} else {
bool foundParam = false;
for (int j = 0; j < argCount; ++j) {
if (node.Parameters[j].Name.Equals(name, StringComparison.Ordinal)) {
newArgs[j] = newArgs[j].Union(args[i]);
foundParam = true;
break;
}
}
if (!foundParam && dictArgsIndex >= 0) {
dictArgs = dictArgs.Union(args[i]);
}
}
}
var limits = unit.ProjectState.Limits;
for (int i = 0; i < argCount; ++i) {
newArgs[i] = ReduceArgs(newArgs[i], limits.NormalArgumentTypes);
}
newArgs[argCount] = ReduceArgs(seqArgs, limits.ListArgumentTypes);
newArgs[argCount + 1] = ReduceArgs(dictArgs, limits.DictArgumentTypes);
return new ArgumentSet(newArgs);
}
private void GetFunctionDescription(FunctionDefinition def, Action<string, VSOBDESCRIPTIONSECTION, IVsNavInfo> addDescription)
{
addDescription(ScopeNode.Name, VSOBDESCRIPTIONSECTION.OBDS_NAME, null);
addDescription("(", VSOBDESCRIPTIONSECTION.OBDS_MISC, null);
for (int i = 0; i < def.Parameters.Count; i++) {
if (i != 0) {
addDescription(", ", VSOBDESCRIPTIONSECTION.OBDS_MISC, null);
}
var curParam = def.Parameters[i];
string name = curParam.Name;
if (curParam.IsDictionary) {
name = "**" + name;
} else if (curParam.IsList) {
name = "*" + curParam.Name;
}
if (curParam.DefaultValue != null) {
// TODO: Support all possible expressions for default values, we should
// probably have a JAst walker for expressions or we should add ToCodeString()
// onto J ASTs so they can round trip
ConstantExpression defaultValue = curParam.DefaultValue as ConstantExpression;
if (defaultValue != null) {
// FIXME: Use j repr
name = name + " = " + defaultValue.ToString(); //JOps.Repr(DefaultContext.Default, defaultValue.Value);
}
}
addDescription(name, VSOBDESCRIPTIONSECTION.OBDS_PARAM, null);
}
addDescription(")\n", VSOBDESCRIPTIONSECTION.OBDS_MISC, null);
}
public override void PostWalk(FunctionDefinition node)
{
_scopes.Add(node);
_allWrites.Add(node.GetVariableReference(_root));
_allWrittenVariables.Add(node.Variable);
base.PostWalk(node);
}
