Example #1
0
        /// <summary>
        /// Returns a new SuiteStatement which is composed of a subset of the statements in this suite statement.
        /// </summary>
        /// <param name="start"></param>
        /// <param name="end"></param>
        /// <returns></returns>
        public SuiteStatement CloneSubset(PythonAst ast, int start, int end)
        {
            Statement[] statements = new Statement[end - start + 1];
            for (int i = start; i <= end; i++)
            {
                statements[i - start] = Statements[i];
            }

            var res = new SuiteStatement(statements);

            // propagate white space so we stay mostly the same...
            var itemWhiteSpace  = this.GetListWhiteSpace(ast);
            var colonWhiteSpace = this.GetProceedingWhiteSpaceDefaultNull(ast);

            if (itemWhiteSpace != null)
            {
                // semi-colon list of statements, must end in a new line, but the original new line
                // could be multiple lines.
                ast.SetAttribute(res, NodeAttributes.ListWhiteSpace, new string[0]);
            }
            else
            {
                ast.SetAttribute(res, NodeAttributes.IsAltFormValue, NodeAttributes.IsAltFormValue);
            }

            return(res);
        }
Example #2
0
        private bool ShouldWalkWorker(SuiteStatement node) {
            if (ShouldWalkWorker((Node)node)) {
                _suites.Add(node);
                foreach (var stmt in node.Statements) {
                    stmt.Walk(this);
                    if (_targetNode != null) {
                        // we have found our extracted code below this,
                        // we should insert before this statement.
                        _insertLocations[_parents[_parents.Count - 1]] = stmt.GetStartIncludingIndentation(_root);
                        break;
                    }
                }
                _suites.Pop();

            }
            return false;
        }
Example #3
0
        private void PostWalkWorker(SuiteStatement node) {
            if (_targetNode == null && node.StartIndex <= _selectedSpan.Start && node.EndIndex >= _selectedSpan.End) {
                // figure out the range of statements we cover...
                int startIndex = 0, endIndex = node.Statements.Count - 1;
                for (int i = 0; i < node.Statements.Count; i++) {
                    if (node.Statements[i].EndIndex >= _selectedSpan.Start) {
                        startIndex = i;
                        break;
                    }
                }
                for (int i = node.Statements.Count - 1; i >= 0; i--) {
                    if (node.Statements[i].StartIndex < _selectedSpan.End) {
                        endIndex = i;
                        break;
                    }
                }
                List<SuiteStatement> followingSuites = new List<SuiteStatement>();
                for (int i = _suites.Count - 1; i >= 0; i--) {
                    if (_suites[i] == null) {
                        // we hit our marker, this is a function/class boundary
                        // We don't care about any suites which come before the marker
                        // because they live in a different scope.  We insert the marker in 
                        // ShouldWalkWorker(Node node) when we have a ScopeStatement.
                        break;
                    }

                    followingSuites.Add(_suites[i]);
                }
                _targetNode = new SuiteTarget(
                    _insertLocations, 
                    _parents.ToArray(), 
                    node, 
                    followingSuites.ToArray(), 
                    _selectedSpan, 
                    startIndex, 
                    endIndex
                );
                _insertLocations[_parents[_parents.Count - 1]] = node.Statements.Count == 0 ?
                    node.GetStartIncludingIndentation(_root) :
                    node.Statements[startIndex].GetStartIncludingIndentation(_root);
            }
        }
Example #4
0
        /// <summary>
        /// Returns a new SuiteStatement which is composed of a subset of the statements in this suite statement.
        /// </summary>
        /// <param name="start"></param>
        /// <param name="end"></param>
        /// <returns></returns>
        public SuiteStatement CloneSubset(PythonAst ast, int start, int end) {
            Statement[] statements = new Statement[end - start + 1];
            for (int i = start; i <= end; i++) {
                statements[i - start] = Statements[i];
            }

            var res = new SuiteStatement(statements);

            // propagate white space so we stay mostly the same...
            var itemWhiteSpace = this.GetListWhiteSpace(ast);
            var colonWhiteSpace = this.GetProceedingWhiteSpaceDefaultNull(ast);

            if (itemWhiteSpace != null) {
                // semi-colon list of statements, must end in a new line, but the original new line
                // could be multiple lines.
                ast.SetAttribute(res, NodeAttributes.ListWhiteSpace, new string[0]);
            } else {
                ast.SetAttribute(res, NodeAttributes.IsAltFormValue, NodeAttributes.IsAltFormValue);
            }

            return res;
        }
Example #5
0
 public override bool Walk(SuiteStatement node)
 {
     node.Parent = _currentScope;
     return(base.Walk(node));
 }
Example #6
0
        /// <summary>
        /// Helper function for calculating all of the drop down entries that are available
        /// in the given suite statement.  Called to calculate both the members of top-level
        /// code and class bodies.
        /// </summary>
        private static ReadOnlyCollection<DropDownEntryInfo> CalculateEntries(SuiteStatement suite) {
            List<DropDownEntryInfo> newEntries = new List<DropDownEntryInfo>();

            if (suite != null) {
                foreach (Statement stmt in suite.Statements) {
                    if (stmt is ClassDefinition || stmt is FunctionDefinition) {
                        newEntries.Add(new DropDownEntryInfo(stmt));
                    }
                }
            }

            newEntries.Sort(ComparisonFunction);
            return new ReadOnlyCollection<DropDownEntryInfo>(newEntries);
        }
Example #7
0
        private static string FindNodeInTree(PythonAst tree, SuiteStatement statement, int line) {
            if (statement != null) {
                foreach (var node in statement.Statements) {
                    FunctionDefinition funcDef = node as FunctionDefinition;
                    if (funcDef != null) {
                        var span = funcDef.GetSpan(tree);
                        if (span.Start.Line <= line && line <= span.End.Line) {
                            var res = FindNodeInTree(tree, funcDef.Body as SuiteStatement, line);
                            if (res != null) {
                                return funcDef.Name + "." + res;
                            }
                            return funcDef.Name;
                        }
                        continue;
                    }

                    ClassDefinition classDef = node as ClassDefinition;
                    if (classDef != null) {
                        var span = classDef.GetSpan(tree);
                        if (span.Start.Line <= line && line <= span.End.Line) {
                            var res = FindNodeInTree(tree, classDef.Body as SuiteStatement, line);
                            if (res != null) {
                                return classDef.Name + "." + res;
                            }
                            return classDef.Name;
                        }
                    }
                }
            }
            return null;
        }
Example #8
0
        private PythonAst ParseFileWorker() {
            StartParsing();

            List<Statement> l = new List<Statement>();

            //
            // A future statement must appear near the top of the module. 
            // The only lines that can appear before a future statement are: 
            // - the module docstring (if any), 
            // - comments, 
            // - blank lines, and 
            // - other future statements. 
            // 

            MaybeEatNewLine();

            if (PeekToken(TokenKind.Constant)) {
                Statement s = ParseStmt();
                l.Add(s);
                _fromFutureAllowed = false;
                ExpressionStatement es = s as ExpressionStatement;
                if (es != null) {
                    ConstantExpression ce = es.Expression as ConstantExpression;
                    if (ce != null && IsString(ce)) {
                        // doc string
                        _fromFutureAllowed = true;
                    }
                }
            }

            MaybeEatNewLine();

            // from __future__
            if (_fromFutureAllowed) {
                while (PeekToken(Tokens.KeywordFromToken)) {
                    Statement s = ParseStmt();
                    l.Add(s);
                    FromImportStatement fis = s as FromImportStatement;
                    if (fis != null && !fis.IsFromFuture) {
                        // end of from __future__
                        break;
                    }
                }
            }

            // the end of from __future__ sequence
            _fromFutureAllowed = false;

            while (true) {
                if (MaybeEatEof()) break;
                if (MaybeEatNewLine()) continue;

                Statement s = ParseStmt();
                l.Add(s);
            }

            Statement[] stmts = l.ToArray();

            SuiteStatement ret = new SuiteStatement(stmts);
            AddIsAltForm(ret);
            if (_token.Token != null) {
                ret.SetLoc(0, GetEnd());
            }
            return CreateAst(ret);
        }
Example #9
0
        //simple_stmt: small_stmt (';' small_stmt)* [';'] Newline
        private Statement ParseSimpleStmt() {
            Statement s = ParseSmallStmt();
            
            string newline = null;
            if (MaybeEat(TokenKind.Semicolon)) {
                var itemWhiteSpace = MakeWhiteSpaceList();
                if (itemWhiteSpace != null) {
                    itemWhiteSpace.Add(_tokenWhiteSpace);
                }

                var start = s.StartIndex;
                List<Statement> l = new List<Statement>();
                l.Add(s);
                while (true) {
                    if (MaybeEatNewLine(out newline) || MaybeEatEof()) {
                        break;
                    }

                    l.Add(ParseSmallStmt());

                    if (MaybeEatEof()) {
                        // implies a new line
                        break;
                    } else if (!MaybeEat(TokenKind.Semicolon)) {
                        EatNewLine(out newline);
                        break;
                    }
                    if (itemWhiteSpace != null) {
                        itemWhiteSpace.Add(_tokenWhiteSpace);
                    }
                }
                Statement[] stmts = l.ToArray();

                SuiteStatement ret = new SuiteStatement(stmts);
                ret.SetLoc(start, stmts[stmts.Length - 1].EndIndex);
                if (itemWhiteSpace != null) {
                    AddListWhiteSpace(ret, itemWhiteSpace.ToArray());
                }
                if (newline != null) {
                    _lookaheadWhiteSpace = newline + _lookaheadWhiteSpace;
                }
                return ret;
            } else if (MaybeEatEof()) {
            } else if (EatNewLine(out newline)) {
                if (_verbatim) {
                    _lookaheadWhiteSpace = newline + _lookaheadWhiteSpace;
                }
            } else {
                // error handling, make sure we're making forward progress
                NextToken();
                if (_verbatim) {
                    _lookaheadWhiteSpace = _tokenWhiteSpace + _token.Token.VerbatimImage + _lookaheadWhiteSpace;
                }
            }
            return s;
        }
Example #10
0
        public ExtractMethodResult GetExtractionResult(ExtractMethodRequest info) {
            bool isStaticMethod = false, isClassMethod = false;
            var parameters = new List<Parameter>();
            string selfParam = null;
            if (info.TargetScope is ClassDefinition) {
                var fromScope = _scopes[_scopes.Length - 1] as FunctionDefinition;
                Debug.Assert(fromScope != null);  // we don't allow extracting from classes, so we have to be coming from a function
                if (fromScope != null) {
                    if (fromScope.Decorators != null) {
                        foreach (var decorator in fromScope.Decorators.Decorators) {
                            NameExpression name = decorator as NameExpression;
                            if (name != null) {
                                if (name.Name == "staticmethod") {
                                    isStaticMethod = true;
                                } else if (name.Name == "classmethod") {
                                    isClassMethod = true;
                                }
                            }
                        }
                    }

                    if (!isStaticMethod) {
                        if (fromScope.Parameters.Count > 0) {
                            selfParam = fromScope.Parameters[0].Name;
                            parameters.Add(new Parameter(selfParam, ParameterKind.Normal));
                        }
                    }
                }
            }

            foreach (var param in info.Parameters) {
                var newParam = new Parameter(param, ParameterKind.Normal);
                if (parameters.Count > 0) {
                    newParam.AddPreceedingWhiteSpace(_ast, " ");
                }
                parameters.Add(newParam);
            }

            // include any non-closed over parameters as well...
            foreach (var input in _inputVars) {
                var variableScope = input.Scope;
                var parentScope = info.TargetScope;

                // are these variables a child of the target scope so we can close over them?
                while (parentScope != null && parentScope != variableScope) {
                    parentScope = parentScope.Parent;
                }

                if (parentScope == null && input.Name != selfParam) {
                    // we can either close over or pass these in as parameters, add them to the list
                    var newParam = new Parameter(input.Name, ParameterKind.Normal);
                    if (parameters.Count > 0) {
                        newParam.AddPreceedingWhiteSpace(_ast, " ");
                    }
                    parameters.Add(newParam);
                }
            }

            var body = _target.GetBody(_ast);
            var isCoroutine = IsCoroutine(body);

            // reset leading indentation to single newline + indentation, this
            // strips out any proceeding comments which we don't extract
            var leading = _newline + body.GetIndentationLevel(_ast);
            body.SetLeadingWhiteSpace(_ast, leading);

            if (_outputVars.Count > 0) {
                // need to add a return statement
                Expression retValue;
                Expression[] names = new Expression[_outputVars.Count];
                int outputIndex = 0;
                foreach (var name in _outputVars) {
                    var nameExpr = new NameExpression(name.Name);
                    nameExpr.AddPreceedingWhiteSpace(_ast, " ");
                    names[outputIndex++] = nameExpr;
                }
                var tuple = new TupleExpression(false, names);
                tuple.RoundTripHasNoParenthesis(_ast);
                retValue = tuple;

                var retStmt = new ReturnStatement(retValue);
                retStmt.SetLeadingWhiteSpace(_ast, leading);

                body = new SuiteStatement(
                    new Statement[] { 
                        body,
                        retStmt
                    }
                );
            } else {
                // we need a SuiteStatement to give us our colon
                body = new SuiteStatement(new Statement[] { body });
            }

            DecoratorStatement decorators = null;
            if (isStaticMethod) {
                decorators = new DecoratorStatement(new[] { new NameExpression("staticmethod") });
            } else if (isClassMethod) {
                decorators = new DecoratorStatement(new[] { new NameExpression("classmethod") });
            }

            var res = new FunctionDefinition(new NameExpression(info.Name), parameters.ToArray(), body, decorators);
            res.IsCoroutine = isCoroutine;
            
            StringBuilder newCall = new StringBuilder();
            newCall.Append(_target.IndentationLevel);
            var method = res.ToCodeString(_ast);

            // fix up indentation...
            for (int curScope = 0; curScope < _scopes.Length; curScope++) {
                if (_scopes[curScope] == info.TargetScope) {
                    // this is our target indentation level.
                    var indentationLevel = _scopes[curScope].Body.GetIndentationLevel(_ast);
                    var lines = method.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None);
                    int minWhiteSpace = Int32.MaxValue;
                    for (int curLine = decorators == null ? 1 : 2; curLine < lines.Length; curLine++) {
                        var line = lines[curLine];

                        for (int i = 0; i < line.Length; i++) {
                            if (!Char.IsWhiteSpace(line[i])) {
                                minWhiteSpace = Math.Min(minWhiteSpace, i);
                                break;
                            }
                        }
                    }

                    StringBuilder newLines = new StringBuilder();
                    newLines.Append(indentationLevel);
                    newLines.Append(lines[0]);
                    if (decorators != null) {
                        newLines.Append(_newline);
                        newLines.Append(indentationLevel);
                        newLines.Append(lines[1]);
                    }

                    // don't include a bunch of blank lines...
                    int endLine = lines.Length - 1;
                    for (; endLine >= 0 && String.IsNullOrWhiteSpace(lines[endLine]); endLine--) {
                    }

                    newLines.Append(_newline);
                    for (int curLine = decorators == null ? 1 : 2; curLine <= endLine; curLine++) {
                        var line = lines[curLine];

                        newLines.Append(indentationLevel);
                        if (_insertTabs) {
                            newLines.Append('\t');
                        } else {
                            newLines.Append(' ', _indentSize);
                        }

                        if (line.Length > minWhiteSpace) {
                            newLines.Append(line, minWhiteSpace, line.Length - minWhiteSpace);
                        }
                        newLines.Append(_newline);
                    }
                    newLines.Append(_newline);
                    method = newLines.ToString();
                    break;
                }
            }

            string comma;
            if (_outputVars.Count > 0) {
                comma = "";
                foreach (var outputVar in _outputVars) {
                    newCall.Append(comma);
                    newCall.Append(outputVar.Name);
                    comma = ", ";
                }
                newCall.Append(" = ");
            } else if (_target.ContainsReturn) {
                newCall.Append("return ");
            }

            if (isCoroutine) {
                newCall.Append("await ");
            }

            if (info.TargetScope is ClassDefinition) {
                var fromScope = _scopes[_scopes.Length - 1] as FunctionDefinition;
                Debug.Assert(fromScope != null);  // we don't allow extracting from classes, so we have to be coming from a function

                if (isStaticMethod) {
                    newCall.Append(info.TargetScope.Name);
                    newCall.Append('.');
                } else if (fromScope != null && fromScope.Parameters.Count > 0) {
                    newCall.Append(fromScope.Parameters[0].Name);
                    newCall.Append('.');
                }
            }

            newCall.Append(info.Name);
            newCall.Append('(');

            comma = "";
            foreach (var param in parameters) {
                if (param.Name != selfParam) {
                    newCall.Append(comma);
                    newCall.Append(param.Name);
                    comma = ", ";
                }
            }

            newCall.Append(')');

            return new ExtractMethodResult(
                method,
                newCall.ToString()
            );
        }
Example #11
0
        //suite: simple_stmt NEWLINE | Newline INDENT stmt+ DEDENT
        private Statement ParseSuite() {

            if (!EatNoEof(TokenKind.Colon)) {
                // improve error handling...
                var error = ErrorStmt(_verbatim ? (_lookaheadWhiteSpace + _lookahead.Token.VerbatimImage) : null);
                NextToken();
                return error;
            }

            string colonWhiteSpace = _tokenWhiteSpace;

            TokenWithSpan cur = _lookahead;
            List<Statement> l = new List<Statement>();

            // we only read a real NewLine here because we need to adjust error reporting
            // for the interpreter.
            SuiteStatement ret;
            if (MaybeEat(TokenKind.NewLine)) {
                string suiteStartWhiteSpace = null;
                if (_verbatim) {
                    suiteStartWhiteSpace = _tokenWhiteSpace + _token.Token.VerbatimImage;
                }

                CheckSuiteEofError(cur);

                // for error reporting we track the NL tokens and report the error on
                // the last one.  This matches CPython.
                cur = _lookahead;
                while (PeekToken(TokenKind.NLToken)) {
                    cur = _lookahead;
                    NextToken();

                    if (_verbatim) {
                        suiteStartWhiteSpace += _tokenWhiteSpace + _token.Token.VerbatimImage;
                    }
                }

                if (!MaybeEat(TokenKind.Indent)) {
                    // no indent?  report the indentation error.
                    if (cur.Token.Kind == TokenKind.Dedent) {
                        ReportSyntaxError(_lookahead.Span.Start, _lookahead.Span.End, "expected an indented block", ErrorCodes.SyntaxError | ErrorCodes.IncompleteStatement);
                    } else {
                        ReportSyntaxError(cur, ErrorCodes.IndentationError);
                    }
                    return ErrorStmt(_verbatim ? (colonWhiteSpace + ':' + suiteStartWhiteSpace) : null);
                } else if (_verbatim) {
                    // indent white space belongs to the statement we're about to parse
                    _lookaheadWhiteSpace = suiteStartWhiteSpace + _tokenWhiteSpace + _token.Token.VerbatimImage +_lookaheadWhiteSpace;
                }

                while (true) {
                    Statement s = ParseStmt();

                    l.Add(s);
                    if (MaybeEat(TokenKind.Dedent)) {
                        // dedent white space belongs to the statement which follows the suite
                        if (_verbatim) {
                            _lookaheadWhiteSpace = _tokenWhiteSpace + _lookaheadWhiteSpace;
                        }
                        break;
                    }
                    if (PeekToken().Kind == TokenKind.EndOfFile) {
                        ReportSyntaxError("unexpected end of file");
                        break; // error handling
                    }
                }
                ret = new SuiteStatement(l.ToArray());
            } else {
                //  simple_stmt NEWLINE
                //  ParseSimpleStmt takes care of the NEWLINE
                ret = new SuiteStatement(new[] { ParseSimpleStmt() });
                if (_verbatim) {
                    AddSecondPreceedingWhiteSpace(ret, "");
                }
            }

            ret.SetLoc(ret.Statements[0].StartIndex, ret.Statements[ret.Statements.Count - 1].EndIndex);
            if (_verbatim) {
                AddPreceedingWhiteSpace(ret, colonWhiteSpace);
            }
            return ret;
        }
Example #12
0
 public SuiteTarget(Dictionary<ScopeStatement, int> insertLocations, ScopeStatement[] parents, SuiteStatement suite, SuiteStatement[] followingSuites, Span selectedSpan, int startIndex, int endIndex)
     : base(insertLocations, parents) {
     _suite = suite;
     _start = startIndex;
     _end = endIndex;
     _followingSuites = followingSuites;
     _selectedSpan = selectedSpan;
 }
Example #13
0
 public override void PostWalk(SuiteStatement node) { PostWalkWorker(node); }
Example #14
0
 public override void PostWalk(SuiteStatement node) {
     while (_scope is StatementScope) {
         _scope = _scope.OuterScope;
     }
     base.PostWalk(node);
 }
Example #15
0
        public override bool Walk(SuiteStatement node) {
            var prevSuite = _curSuite;
            _curSuite = node;

            // recursively walk the statements in the suite
            if (node.Statements != null) {
                foreach (var innerNode in node.Statements) {
                    innerNode.Walk(this);
                }
            }

            _curSuite = prevSuite;

            // then check if we encountered an assert which added an isinstance scope.
            IsInstanceScope isInstanceScope = _scope as IsInstanceScope;
            if (isInstanceScope != null && isInstanceScope._effectiveSuite == node) {
                // pop the isinstance scope
                _scope = _scope.OuterScope;
                // transform back into a line number and start the new statement scope on the line
                // after the suite statement.
                var lineNo = _tree.IndexToLocation(node.EndIndex).Line;

                int offset;
                if (_tree._lineLocations.Length == 0) {
                    // single line input
                    offset = 0;
                } else {
                    offset = lineNo < _tree._lineLocations.Length ? _tree._lineLocations[lineNo].EndIndex : _tree._lineLocations[_tree._lineLocations.Length - 1].EndIndex;
                }
                var closingScope = new StatementScope(offset, _scope);
                _scope.Children.Add(closingScope);
                _scope = closingScope;
            }
            return false;
        }
Example #16
0
        private void PushIsInstanceScope(Node node, KeyValuePair<NameExpression, Expression>[] isInstanceNames, SuiteStatement effectiveSuite) {
            InterpreterScope scope;
            if (!_curUnit.Scope.TryGetNodeScope(node, out scope)) {
                // find our parent scope, it may not be just the last entry in _scopes
                // because that can be a StatementScope and we would start a new range.
                var declScope = _scope.EnumerateTowardsGlobal.FirstOrDefault(s => !(s is StatementScope));

                scope = new IsInstanceScope(node.StartIndex, effectiveSuite, declScope);

                declScope.Children.Add(scope);
                declScope.AddNodeScope(node, scope);
                _scope = scope;
            }
        }
Example #17
0
 private static string GetDoc(SuiteStatement node) {
     var docExpr = node?.Statements?.FirstOrDefault() as ExpressionStatement;
     var ce = docExpr?.Expression as ConstantExpression;
     return ce?.Value as string;
 }
Example #18
0
            public override bool Walk(SuiteStatement node) {
                foreach (var statement in node.Statements) {
                    if (statement is BreakStatement || statement is ContinueStatement) {
                        // code after here is unreachable
                        break;
                    }

                    Returns = false;
                    statement.Walk(this);
                    if (Returns) {
                        // rest of the code is unreachable...
                        break;
                    }
                }
                return false;
            }
Example #19
0
 public IsInstanceScope(int startIndex, SuiteStatement effectiveSuite, InterpreterScope outerScope)
     : base(null, null, outerScope) {
     _startIndex = _endIndex = startIndex;
     _effectiveSuite = effectiveSuite;
 }
Example #20
0
 // SuiteStatement
 public override bool Walk(SuiteStatement node) { return ShouldWalkWorker(node); }