/// <summary>
/// Returns a new SuiteStatement which is composed of a subset of the statements in this suite statement.
/// </summary>
/// <param name="start"></param>
/// <param name="end"></param>
/// <returns></returns>
public SuiteStatement CloneSubset(JAst ast, int start, int end)
{
Statement[] statements = new Statement[end - start + 1];
for (int i = start; i <= end; i++) {
statements[i - start] = Statements[i];
}
var res = new SuiteStatement(statements);
// propagate white space so we stay mostly the same...
var itemWhiteSpace = this.GetListWhiteSpace(ast);
var colonWhiteSpace = this.GetProceedingWhiteSpaceDefaultNull(ast);
if (colonWhiteSpace != null) {
ast.SetAttribute(res, NodeAttributes.PreceedingWhiteSpace, "");
} else if (itemWhiteSpace != null) {
// semi-colon list of statements, must end in a new line, but the original new line
// could be multiple lines.
ast.SetAttribute(res, NodeAttributes.ListWhiteSpace, new string[0]);
var trailingNewLine = this.GetTrailingNewLine(ast);
if (trailingNewLine != null) {
ast.SetAttribute(res, NodeAttributes.TrailingNewLine, "\r\n");
}
}
if (this.IsAltForm(ast)) {
ast.SetAttribute(res, NodeAttributes.IsAltFormValue, NodeAttributes.IsAltFormValue);
}
return res;
}
//suite: simple_stmt NEWLINE | Newline INDENT stmt+ DEDENT
private Statement ParseSuite()
{
if (!EatNoEof(TokenKind.Colon)) {
// improve error handling...
var error = ErrorStmt(_verbatim ? (_lookaheadWhiteSpace + _lookahead.Token.VerbatimImage) : null);
NextToken();
return error;
}
string colonWhiteSpace = _tokenWhiteSpace;
TokenWithSpan cur = _lookahead;
List<Statement> l = new List<Statement>();
// we only read a real NewLine here because we need to adjust error reporting
// for the interpreter.
SuiteStatement ret;
if (MaybeEat(TokenKind.NewLine)) {
string suiteStartWhiteSpace = null;
if (_verbatim) {
suiteStartWhiteSpace = _tokenWhiteSpace + _token.Token.VerbatimImage;
}
CheckSuiteEofError(cur);
// for error reporting we track the NL tokens and report the error on
// the last one. This matches CJ.
cur = _lookahead;
while (PeekToken(TokenKind.NLToken)) {
cur = _lookahead;
NextToken();
if (_verbatim) {
suiteStartWhiteSpace += _tokenWhiteSpace + _token.Token.VerbatimImage;
}
}
if (!MaybeEat(TokenKind.Indent)) {
// no indent? report the indentation error.
if (cur.Token.Kind == TokenKind.Dedent) {
ReportSyntaxError(_lookahead.Span.Start, _lookahead.Span.End, "expected an indented block", ErrorCodes.SyntaxError | ErrorCodes.IncompleteStatement);
} else {
ReportSyntaxError(cur, ErrorCodes.IndentationError);
}
return ErrorStmt(_verbatim ? (colonWhiteSpace + ':' + suiteStartWhiteSpace) : null);
} else if (_verbatim) {
// indent white space belongs to the statement we're about to parse
_lookaheadWhiteSpace = _tokenWhiteSpace + _token.Token.VerbatimImage +_lookaheadWhiteSpace;
}
string trailingWhiteSpace = null;
while (true) {
Statement s = ParseStmt();
l.Add(s);
if (MaybeEat(TokenKind.Dedent)) {
// dedent white space belongs to the statement which follows the suite
if (_verbatim) {
_lookaheadWhiteSpace = _tokenWhiteSpace + _lookaheadWhiteSpace;
}
break;
}
if (PeekToken().Kind == TokenKind.EndOfFile) {
ReportSyntaxError("unexpected end of file");
break; // error handling
}
}
ret = new SuiteStatement(l.ToArray());
if (_verbatim) {
AddSecondPreceedingWhiteSpace(ret, suiteStartWhiteSpace);
if (trailingWhiteSpace != null) {
AddTrailingNewLine(ret, trailingWhiteSpace);
}
}
} else {
// simple_stmt NEWLINE
// ParseSimpleStmt takes care of the NEWLINE
ret = new SuiteStatement(new[] { ParseSimpleStmt() });
if (_verbatim) {
AddSecondPreceedingWhiteSpace(ret, "");
}
}
ret.SetLoc(ret.Statements[0].StartIndex, ret.Statements[ret.Statements.Count - 1].EndIndex);
if (_verbatim) {
AddPreceedingWhiteSpace(ret, colonWhiteSpace);
}
return ret;
}
//simple_stmt: small_stmt (';' small_stmt)* [';'] Newline
private Statement ParseSimpleStmt()
{
Statement s = ParseSmallStmt();
string newline = null;
if (MaybeEat(TokenKind.Semicolon)) {
var itemWhiteSpace = MakeWhiteSpaceList();
if (itemWhiteSpace != null) {
itemWhiteSpace.Add(_tokenWhiteSpace);
}
var start = s.StartIndex;
List<Statement> l = new List<Statement>();
l.Add(s);
while (true) {
if (MaybeEatNewLine(out newline) || MaybeEatEof()) {
break;
}
l.Add(ParseSmallStmt());
if (MaybeEatEof()) {
// implies a new line
break;
} else if (!MaybeEat(TokenKind.Semicolon)) {
EatNewLine(out newline);
break;
}
if (itemWhiteSpace != null) {
itemWhiteSpace.Add(_tokenWhiteSpace);
}
}
Statement[] stmts = l.ToArray();
SuiteStatement ret = new SuiteStatement(stmts);
ret.SetLoc(start, stmts[stmts.Length - 1].EndIndex);
if (itemWhiteSpace != null) {
AddListWhiteSpace(ret, itemWhiteSpace.ToArray());
}
if (newline != null) {
AddTrailingNewLine(ret, newline);
}
return ret;
} else if (MaybeEatEof()) {
} else if (EatNewLine(out newline)) {
if (_verbatim) {
AddTrailingNewLine(s, newline);
}
} else {
// error handling, make sure we're making forward progress
NextToken();
if (_verbatim) {
AddTrailingNewLine(s, _tokenWhiteSpace + _token.Token.VerbatimImage);
}
}
return s;
}
private JAst ParseFileWorker()
{
StartParsing();
List<Statement> l = new List<Statement>();
//
// A future statement must appear near the top of the module.
// The only lines that can appear before a future statement are:
// - the module docstring (if any),
// - comments,
// - blank lines, and
// - other future statements.
//
MaybeEatNewLine();
if (PeekToken(TokenKind.Constant)) {
Statement s = ParseStmt();
l.Add(s);
_fromFutureAllowed = false;
ExpressionStatement es = s as ExpressionStatement;
if (es != null) {
ConstantExpression ce = es.Expression as ConstantExpression;
if (ce != null && IsString(ce)) {
// doc string
_fromFutureAllowed = true;
}
}
}
MaybeEatNewLine();
// from __future__
if (_fromFutureAllowed) {
while (PeekToken(Tokens.KeywordFromToken)) {
Statement s = ParseStmt();
l.Add(s);
FromImportStatement fis = s as FromImportStatement;
if (fis != null && !fis.IsFromFuture) {
// end of from __future__
break;
}
}
}
// the end of from __future__ sequence
_fromFutureAllowed = false;
while (true) {
if (MaybeEatEof()) break;
if (MaybeEatNewLine()) continue;
Statement s = ParseStmt();
l.Add(s);
}
Statement[] stmts = l.ToArray();
SuiteStatement ret = new SuiteStatement(stmts);
AddIsAltForm(ret);
if (_token.Token != null) {
ret.SetLoc(0, GetEnd());
}
return CreateAst(ret);
}
/// <summary>
/// Helper function for calculating all of the drop down entries that are available
/// in the given suite statement. Called to calculate both the members of top-level
/// code and class bodies.
/// </summary>
private static ReadOnlyCollection<DropDownEntryInfo> CalculateEntries(SuiteStatement suite)
{
List<DropDownEntryInfo> newEntries = new List<DropDownEntryInfo>();
if (suite != null) {
foreach (Statement stmt in suite.Statements) {
if (stmt is ClassDefinition || stmt is FunctionDefinition) {
newEntries.Add(new DropDownEntryInfo(stmt));
}
}
}
newEntries.Sort(ComparisonFunction);
return new ReadOnlyCollection<DropDownEntryInfo>(newEntries);
}
public IsInstanceScope(int startIndex, SuiteStatement effectiveSuite, InterpreterScope outerScope)
: base(null, null, outerScope)
{
_startIndex = _endIndex = startIndex;
_effectiveSuite = effectiveSuite;
}
public override bool Walk(SuiteStatement node)
{
foreach (var statement in node.Statements) {
if (statement is BreakStatement || statement is ContinueStatement) {
// code after here is unreachable
break;
}
Returns = false;
statement.Walk(this);
if (Returns) {
// rest of the code is unreachable...
break;
}
}
return false;
}
public override Statement GetBody(JAst root)
{
if (_suite.Statements.Count == 0) {
return _suite;
}
var ast = _suite.CloneSubset(Parents[0] as JAst, _start, _end);
if (!_suite.IsFunctionOrClassSuite(root)) {
ast = new SuiteStatement(new[] { ast });
}
return ast;
}
public override bool Walk(SuiteStatement node)
{
var prevSuite = _curSuite;
_curSuite = node;
// recursively walk the statements in the suite
if (node.Statements != null) {
foreach (var innerNode in node.Statements) {
innerNode.Walk(this);
}
}
_curSuite = prevSuite;
// then check if we encountered an assert which added an isinstance scope.
IsInstanceScope isInstanceScope = _scope as IsInstanceScope;
if (isInstanceScope != null && isInstanceScope._effectiveSuite == node) {
// pop the isinstance scope
_scope = _scope.OuterScope;
var declScope = _curUnit.Scope;
// transform back into a line number and start the new statement scope on the line
// after the suite statement.
var lineNo = _entry.Tree.IndexToLocation(node.EndIndex).Line;
int offset;
if (_entry.Tree._lineLocations.Length == 0) {
// single line input
offset = 0;
} else {
offset = lineNo < _entry.Tree._lineLocations.Length ? _entry.Tree._lineLocations[lineNo] : _entry.Tree._lineLocations[_entry.Tree._lineLocations.Length - 1];
}
var closingScope = new StatementScope(offset, declScope);
_scope = closingScope;
declScope.Children.Add(closingScope);
}
return false;
}
public override void PostWalk(SuiteStatement node)
{
while (_scope is StatementScope) {
_scope = _scope.OuterScope;
}
base.PostWalk(node);
}
private bool ShouldWalkWorker(SuiteStatement node)
{
if (ShouldWalkWorker((Node)node)) {
_suites.Add(node);
foreach (var stmt in node.Statements) {
stmt.Walk(this);
if (_targetNode != null) {
// we have found our extracted code below this,
// we should insert before this statement.
_insertLocations[_parents[_parents.Count - 1]] = stmt.GetStartIncludingWhiteSpace(_root);
break;
}
}
_suites.Pop();
}
return false;
}
private void PostWalkWorker(SuiteStatement node)
{
if (_targetNode == null && node.StartIndex <= _selectedSpan.Start && node.EndIndex >= _selectedSpan.End) {
// figure out the range of statements we cover...
int startIndex = 0, endIndex = node.Statements.Count - 1;
for (int i = 0; i < node.Statements.Count; i++) {
if (node.Statements[i].EndIndex >= _selectedSpan.Start) {
startIndex = i;
break;
}
}
for (int i = node.Statements.Count - 1; i >= 0; i--) {
if (node.Statements[i].StartIndex < _selectedSpan.End) {
endIndex = i;
break;
}
}
List<SuiteStatement> followingSuites = new List<SuiteStatement>();
for (int i = _suites.Count - 1; i >= 0; i--) {
if (_suites[i] == null) {
// we hit our marker, this is a function/class boundary
// We don't care about any suites which come before the marker
// because they live in a different scope. We insert the marker in
// ShouldWalkWorker(Node node) when we have a ScopeStatement.
break;
}
followingSuites.Add(_suites[i]);
}
_targetNode = new SuiteTarget(
_insertLocations,
_parents.ToArray(),
node,
followingSuites.ToArray(),
_selectedSpan,
startIndex,
endIndex
);
_insertLocations[_parents[_parents.Count - 1]] = node.Statements.Count == 0 ?
node.GetStartIncludingWhiteSpace(_root) :
node.Statements[startIndex].GetStartIncludingWhiteSpace(_root);
}
}
// SuiteStatement
public override bool Walk(SuiteStatement node)
{
return ShouldWalkWorker(node);
}
public override void PostWalk(SuiteStatement node)
{
PostWalkWorker(node);
}
private void PushIsInstanceScope(Node node, KeyValuePair<NameExpression, Expression>[] isInstanceNames, SuiteStatement effectiveSuite)
{
InterpreterScope scope;
if (!_curUnit.Scope.TryGetNodeScope(node, out scope)) {
if (_scope is IsInstanceScope) {
// Reuse the current scope
_curUnit.Scope.AddNodeScope(node, _scope);
return;
}
// find our parent scope, it may not be just the last entry in _scopes
// because that can be a StatementScope and we would start a new range.
var declScope = _scope.EnumerateTowardsGlobal.FirstOrDefault(s => !(s is StatementScope));
scope = new IsInstanceScope(node.StartIndex, effectiveSuite, declScope);
declScope.Children.Add(scope);
declScope.AddNodeScope(node, scope);
_scope = scope;
}
}
public SuiteTarget(Dictionary<ScopeStatement, int> insertLocations, ScopeStatement[] parents, SuiteStatement suite, SuiteStatement[] followingSuites, Span selectedSpan, int startIndex, int endIndex)
: base(insertLocations, parents)
{
_suite = suite;
_start = startIndex;
_end = endIndex;
_followingSuites = followingSuites;
_selectedSpan = selectedSpan;
}
public ExtractMethodResult GetExtractionResult(ExtractMethodRequest info)
{
bool isStaticMethod = false, isClassMethod = false;
var parameters = new List<Parameter>();
string selfParam = null;
if (info.TargetScope is ClassDefinition) {
var fromScope = _scopes[_scopes.Length - 1] as FunctionDefinition;
Debug.Assert(fromScope != null); // we don't allow extracting from classes, so we have to be coming from a function
if (fromScope.Decorators != null) {
foreach (var decorator in fromScope.Decorators.Decorators) {
NameExpression name = decorator as NameExpression;
if (name != null) {
if (name.Name == "staticmethod") {
isStaticMethod = true;
} else if (name.Name == "classmethod") {
isClassMethod = true;
}
}
}
}
if (!isStaticMethod) {
if (fromScope.Parameters.Count > 0) {
selfParam = fromScope.Parameters[0].Name;
parameters.Add(new Parameter(selfParam, ParameterKind.Normal));
}
}
}
foreach (var param in info.Parameters) {
var newParam = new Parameter(param, ParameterKind.Normal);
if (parameters.Count > 0) {
newParam.AddPreceedingWhiteSpace(_ast, " ");
}
parameters.Add(newParam);
}
// include any non-closed over parameters as well...
foreach (var input in _inputVars) {
var variableScope = input.Scope;
var parentScope = info.TargetScope;
// are these variables a child of the target scope so we can close over them?
while (parentScope != null && parentScope != variableScope) {
parentScope = parentScope.Parent;
}
if (parentScope == null && input.Name != selfParam) {
// we can either close over or pass these in as parameters, add them to the list
var newParam = new Parameter(input.Name, ParameterKind.Normal);
if (parameters.Count > 0) {
newParam.AddPreceedingWhiteSpace(_ast, " ");
}
parameters.Add(newParam);
}
}
var body = _target.GetBody(_ast);
if (_outputVars.Count > 0) {
// need to add a return statement
Expression retValue;
Expression[] names = new Expression[_outputVars.Count];
int outputIndex = 0;
foreach (var name in _outputVars) {
var nameExpr = new NameExpression(name.Name);
nameExpr.AddPreceedingWhiteSpace(_ast, " ");
names[outputIndex++] = nameExpr;
}
var tuple = new TupleExpression(false, names);
tuple.RoundTripHasNoParenthesis(_ast);
retValue = tuple;
var retStmt = new ReturnStatement(retValue);
if (body is SuiteStatement) {
SuiteStatement suite = (SuiteStatement)body;
Node.CopyLeadingWhiteSpace(_ast, suite.Statements[0], retStmt);
Node.CopyTrailingNewLine(_ast, suite.Statements[0], suite.Statements[suite.Statements.Count - 1]);
Statement[] statements = new Statement[suite.Statements.Count + 1];
for (int i = 0; i < suite.Statements.Count; i++) {
statements[i] = suite.Statements[i];
}
statements[statements.Length - 1] = retStmt;
body = new SuiteStatement(statements);
} else {
Node.CopyLeadingWhiteSpace(_ast, body, retStmt);
body = new SuiteStatement(
new Statement[] {
body,
retStmt
}
);
}
}
DecoratorStatement decorators = null;
if (isStaticMethod) {
decorators = new DecoratorStatement(new[] { new NameExpression("staticmethod") });
} else if (isClassMethod) {
decorators = new DecoratorStatement(new[] { new NameExpression("classmethod") });
}
var res = new FunctionDefinition(new NameExpression(info.Name), parameters.ToArray(), body, decorators);
StringBuilder newCall = new StringBuilder();
newCall.Append(_target.IndentationLevel);
var method = res.ToCodeString(_ast);
// fix up indentation...
for (int curScope = 0; curScope < _scopes.Length; curScope++) {
if (_scopes[curScope] == info.TargetScope) {
// this is our target indentation level.
var indentationLevel = _scopes[curScope].Body.GetIndentationLevel(_ast);
var lines = method.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None);
int minWhiteSpace = Int32.MaxValue;
for (int curLine = decorators == null ? 1 : 2; curLine < lines.Length; curLine++) {
var line = lines[curLine];
for (int i = 0; i < line.Length; i++) {
if (!Char.IsWhiteSpace(line[i])) {
minWhiteSpace = Math.Min(minWhiteSpace, i);
break;
}
}
}
StringBuilder newLines = new StringBuilder();
newLines.Append(indentationLevel);
newLines.Append(lines[0]);
if (decorators != null) {
newLines.Append("\r\n");
newLines.Append(indentationLevel);
newLines.Append(lines[1]);
}
// don't include a bunch of blank lines...
int endLine = lines.Length - 1;
for (; endLine >= 0 && String.IsNullOrWhiteSpace(lines[endLine]); endLine--) {
}
newLines.Append("\r\n");
for (int curLine = decorators == null ? 1 : 2; curLine <= endLine; curLine++) {
var line = lines[curLine];
newLines.Append(indentationLevel);
if (_insertTabs) {
newLines.Append('\t');
} else {
newLines.Append(' ', _indentSize);
}
if (line.Length > minWhiteSpace) {
newLines.Append(line, minWhiteSpace, line.Length - minWhiteSpace);
}
newLines.Append("\r\n");
}
newLines.Append("\r\n");
method = newLines.ToString();
break;
}
}
string comma;
if (_outputVars.Count > 0) {
comma = "";
foreach (var outputVar in _outputVars) {
newCall.Append(comma);
newCall.Append(outputVar.Name);
comma = ", ";
}
newCall.Append(" = ");
} else if (_target.ContainsReturn) {
newCall.Append("return ");
}
if (info.TargetScope is ClassDefinition) {
var fromScope = _scopes[_scopes.Length - 1] as FunctionDefinition;
Debug.Assert(fromScope != null); // we don't allow extracting from classes, so we have to be coming from a function
if (isStaticMethod) {
newCall.Append(info.TargetScope.Name);
newCall.Append('.');
} else if (fromScope.Parameters.Count > 0) {
newCall.Append(fromScope.Parameters[0].Name);
newCall.Append('.');
}
}
newCall.Append(info.Name);
newCall.Append('(');
comma = "";
foreach (var param in parameters) {
if (param.Name != selfParam) {
newCall.Append(comma);
newCall.Append(param.Name);
comma = ", ";
}
}
newCall.Append(')');
return new ExtractMethodResult(
method,
newCall.ToString()
);
}
public override IEnumerable<Statement> GetStatementsAfter(JAst root)
{
var ast = _suite.CloneSubset(Parents[0] as JAst, _end + 1, _suite.Statements.Count - 1);
if (!_suite.IsFunctionOrClassSuite(root)) {
ast = new SuiteStatement(new[] { ast });
}
yield return ast;
foreach (var suite in _followingSuites) {
foreach (var stmt in suite.Statements) {
if (stmt.StartIndex > _selectedSpan.End) {
yield return stmt;
}
}
}
}
private static string FindNodeInTree(JAst tree, SuiteStatement statement, int line)
{
if (statement != null) {
foreach (var node in statement.Statements) {
FunctionDefinition funcDef = node as FunctionDefinition;
if (funcDef != null) {
var span = funcDef.GetSpan(tree);
if (span.Start.Line <= line && line <= span.End.Line) {
var res = FindNodeInTree(tree, funcDef.Body as SuiteStatement, line);
if (res != null) {
return funcDef.Name + "." + res;
}
return funcDef.Name;
}
continue;
}
ClassDefinition classDef = node as ClassDefinition;
if (classDef != null) {
var span = classDef.GetSpan(tree);
if (span.Start.Line <= line && line <= span.End.Line) {
var res = FindNodeInTree(tree, classDef.Body as SuiteStatement, line);
if (res != null) {
return classDef.Name + "." + res;
}
return classDef.Name;
}
}
}
}
return null;
}