public void KeywordClassification27()
{
var code = string.Join(Environment.NewLine, PythonKeywords.All(PythonLanguageVersion.V27));
code += "\r\nTrue\r\nFalse";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
foreach (var span in helper.AstClassifierSpans)
{
var text = span.Span.GetText();
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
// None, True and False are special
if (text == "None" || text == "True" || text == "False")
{
Assert.AreEqual("Python builtin", span.ClassificationType.Classification, text);
continue;
}
Assert.AreEqual("keyword", span.ClassificationType.Classification, text);
}
}
}
internal bool IsInGrouping()
{
// We assume that groupings are correctly matched and keep a simple
// nesting count.
int nesting = 0;
foreach (var token in this)
{
if (token == null)
{
continue;
}
if (token.IsCloseGrouping())
{
nesting++;
}
else if (token.IsOpenGrouping())
{
if (nesting-- == 0)
{
return(true);
}
}
else if (token.ClassificationType.IsOfType(PredefinedClassificationTypeNames.Keyword) &&
PythonKeywords.IsOnlyStatementKeyword(token.Span.GetText()))
{
return(false);
}
}
return(false);
}
/// <summary>
/// Walks backwards to figure out if we're a parameter name which comes after a comma.
/// </summary>
private bool IsParameterNameComma(IEnumerator <ClassificationSpan> enumerator)
{
int groupingLevel = 1;
while (MoveNextSkipExplicitNewLines(enumerator))
{
if (enumerator.Current.ClassificationType == Classifier.Provider.Keyword)
{
if (enumerator.Current.Span.GetText() == "def" && groupingLevel == 0)
{
return(true);
}
if (PythonKeywords.IsOnlyStatementKeyword(enumerator.Current.Span.GetText()))
{
return(false);
}
}
if (enumerator.Current.IsOpenGrouping())
{
groupingLevel--;
if (groupingLevel == 0)
{
return(IsParameterNameOpenParen(enumerator));
}
}
else if (enumerator.Current.IsCloseGrouping())
{
groupingLevel++;
}
}
return(false);
}
internal static bool IsValidPythonIdentifier(string identifier, PythonLanguageVersion pythonVersion)
{
if (String.IsNullOrEmpty(identifier) || PythonKeywords.IsKeyword(identifier, pythonVersion))
{
return(false);
}
//Python2 identifiers are only certain ASCII characters
if (pythonVersion < PythonLanguageVersion.V30)
{
return(Python2IdentifierRegex.IsMatch(identifier));
}
//Python3 identifiers can include unicode characters
if (!Tokenizer.IsIdentifierStartChar(identifier[0]))
{
return(false);
}
return(identifier.Skip(1).All(Tokenizer.IsIdentifierChar));
}
internal static bool IsInGrouping(ITextSnapshot snapshot, IEnumerable <TrackingTokenInfo> tokensInReverse)
{
int nesting = 0;
foreach (var token in tokensInReverse)
{
if (token.Category == Parsing.TokenCategory.Grouping)
{
var t = token.GetText(snapshot);
if (t.IsCloseGrouping())
{
nesting++;
}
else if (t.IsOpenGrouping())
{
if (nesting-- == 0)
{
return(true);
}
}
}
else if (token.Category == Parsing.TokenCategory.Delimiter)
{
if (nesting == 0 && token.GetText(snapshot) == ",")
{
return(true);
}
}
else if (token.Category == Parsing.TokenCategory.Keyword)
{
if (PythonKeywords.IsOnlyStatementKeyword(token.GetText(snapshot)))
{
return(false);
}
}
}
return(false);
}
public void KeywordClassification33()
{
var code = string.Join(Environment.NewLine, PythonKeywords.All(PythonLanguageVersion.V33));
using (var helper = new ClassifierHelper(MockTextBuffer(code), PythonLanguageVersion.V33)) {
foreach (var span in helper.AstClassifierSpans)
{
var text = span.Span.GetText();
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
// None is special
if (text == "None")
{
Assert.AreEqual("Python builtin", span.ClassificationType.Classification, text);
continue;
}
Assert.AreEqual("keyword", span.ClassificationType.Classification, text);
}
}
}
private static int CalculateIndentation(string baseline, ITextSnapshotLine line, IEditorOptions options, IClassifier classifier, ITextView textView)
{
int indentation = GetIndentation(baseline, options.GetTabSize());
int tabSize = options.GetIndentSize();
var tokens = classifier.GetClassificationSpans(line.Extent);
if (tokens.Count > 0 && !IsUnterminatedStringToken(tokens[tokens.Count - 1]))
{
int tokenIndex = tokens.Count - 1;
while (tokenIndex >= 0 &&
(tokens[tokenIndex].ClassificationType.IsOfType(PredefinedClassificationTypeNames.Comment) ||
tokens[tokenIndex].ClassificationType.IsOfType(PredefinedClassificationTypeNames.WhiteSpace)))
{
tokenIndex--;
}
if (tokenIndex < 0)
{
return(indentation);
}
if (ReverseExpressionParser.IsExplicitLineJoin(tokens[tokenIndex]))
{
// explicit line continuation, we indent 1 level for the continued line unless
// we're already indented because of multiple line continuation characters.
indentation = GetIndentation(line.GetText(), options.GetTabSize());
var joinedLine = tokens[tokenIndex].Span.Start.GetContainingLine();
if (joinedLine.LineNumber > 0)
{
var prevLineSpans = classifier.GetClassificationSpans(tokens[tokenIndex].Span.Snapshot.GetLineFromLineNumber(joinedLine.LineNumber - 1).Extent);
if (prevLineSpans.Count == 0 || !ReverseExpressionParser.IsExplicitLineJoin(prevLineSpans[prevLineSpans.Count - 1]))
{
indentation += tabSize;
}
}
else
{
indentation += tabSize;
}
return(indentation);
}
string sline = tokens[tokenIndex].Span.GetText();
var lastChar = sline.Length == 0 ? '\0' : sline[sline.Length - 1];
// use the expression parser to figure out if we're in a grouping...
var spans = textView.BufferGraph.MapDownToFirstMatch(
tokens[tokenIndex].Span,
SpanTrackingMode.EdgePositive,
PythonContentTypePrediciate
);
if (spans.Count == 0)
{
return(indentation);
}
var revParser = new ReverseExpressionParser(
spans[0].Snapshot,
spans[0].Snapshot.TextBuffer,
spans[0].Snapshot.CreateTrackingSpan(
spans[0].Span,
SpanTrackingMode.EdgePositive
)
);
var tokenStack = new System.Collections.Generic.Stack <ClassificationSpan>();
tokenStack.Push(null); // end with an implicit newline
bool endAtNextNull = false;
foreach (var token in revParser)
{
tokenStack.Push(token);
if (token == null && endAtNextNull)
{
break;
}
else if (token != null &&
token.ClassificationType == revParser.Classifier.Provider.Keyword &&
PythonKeywords.IsOnlyStatementKeyword(token.Span.GetText()))
{
endAtNextNull = true;
}
}
var indentStack = new System.Collections.Generic.Stack <LineInfo>();
var current = LineInfo.Empty;
while (tokenStack.Count > 0)
{
var token = tokenStack.Pop();
if (token == null)
{
current.NeedsUpdate = true;
}
else if (token.IsOpenGrouping())
{
indentStack.Push(current);
var start = token.Span.Start;
var line2 = start.GetContainingLine();
var next = tokenStack.Count > 0 ? tokenStack.Peek() : null;
if (next != null && next.Span.End <= line2.End)
{
current = new LineInfo {
Indentation = start.Position - line2.Start.Position + 1
};
}
else
{
current = new LineInfo {
Indentation = GetIndentation(line2.GetText(), tabSize) + tabSize
};
}
}
else if (token.IsCloseGrouping())
{
if (indentStack.Count > 0)
{
current = indentStack.Pop();
}
else
{
current.NeedsUpdate = true;
}
}
else if (ReverseExpressionParser.IsExplicitLineJoin(token))
{
while (token != null && tokenStack.Count > 0)
{
token = tokenStack.Pop();
}
}
else if (current.NeedsUpdate == true)
{
var line2 = token.Span.Start.GetContainingLine();
current = new LineInfo {
Indentation = GetIndentation(line2.GetText(), tabSize)
};
}
if (token != null && ShouldDedentAfterKeyword(token)) // dedent after some statements
{
current.ShouldDedentAfter = true;
}
if (token != null && token.Span.GetText() == ":" && // indent after a colon
indentStack.Count == 0) // except in a grouping
{
current.ShouldIndentAfter = true;
// If the colon isn't at the end of the line, cancel it out.
// If the following is a ShouldDedentAfterKeyword, only one dedent will occur.
current.ShouldDedentAfter = (tokenStack.Count != 0 && tokenStack.Peek() != null);
}
}
indentation = current.Indentation +
(current.ShouldIndentAfter ? tabSize : 0) -
(current.ShouldDedentAfter ? tabSize : 0);
}
// Map indentation back to the view's text buffer.
int offset = 0;
var viewLineStart = textView.BufferGraph.MapUpToSnapshot(line.Start, PointTrackingMode.Positive, PositionAffinity.Successor, textView.TextSnapshot);
if (viewLineStart.HasValue)
{
offset = viewLineStart.Value.Position - viewLineStart.Value.GetContainingLine().Start.Position;
}
return(offset + indentation);
}
private static int CalculateIndentation(
string baseline,
ITextSnapshotLine line,
IEditorOptions options,
PythonTextBufferInfo buffer
)
{
var snapshot = line.Snapshot;
if (snapshot.TextBuffer != buffer.Buffer)
{
throw new ArgumentException("buffer mismatch");
}
int indentation = GetIndentation(baseline, options.GetTabSize());
int tabSize = options.GetIndentSize();
var tokens = buffer.GetTokens(line).ToList();
while (tokens.Count > 0 && IsWhitespace(tokens[tokens.Count - 1].Category))
{
tokens.RemoveAt(tokens.Count - 1);
}
if (tokens.Count == 0 || IsUnterminatedStringToken(tokens[tokens.Count - 1], snapshot))
{
return(indentation);
}
if (HasExplicitLineJoin(tokens, snapshot))
{
// explicit line continuation, we indent 1 level for the continued line unless
// we're already indented because of multiple line continuation characters.
indentation = GetIndentation(line.GetText(), options.GetTabSize());
var joinedLine = line.LineNumber - 1;
if (joinedLine >= 0)
{
var prevLineTokens = buffer.GetTokens(snapshot.GetLineFromLineNumber(joinedLine)).ToList();
if (prevLineTokens.Count == 0 || !HasExplicitLineJoin(prevLineTokens, snapshot))
{
indentation += tabSize;
}
}
else
{
indentation += tabSize;
}
return(indentation);
}
var tokenStack = new Stack <TrackingTokenInfo?>();
tokenStack.Push(null); // end with an implicit newline
int endAtLine = -1, currentLine = tokens.Last().LineNumber;
foreach (var t in buffer.GetTokensInReverseFromPoint(tokens.Last().ToSnapshotSpan(snapshot).Start))
{
if (t.LineNumber == currentLine)
{
tokenStack.Push(t);
}
else
{
tokenStack.Push(null);
}
if (t.LineNumber == endAtLine)
{
break;
}
else if (t.Category == TokenCategory.Keyword && PythonKeywords.IsOnlyStatementKeyword(t.GetText(snapshot), buffer.LanguageVersion))
{
endAtLine = t.LineNumber - 1;
}
if (t.LineNumber != currentLine)
{
currentLine = t.LineNumber;
if (t.Category != TokenCategory.WhiteSpace && t.Category != TokenCategory.Comment && t.Category != TokenCategory.LineComment)
{
tokenStack.Push(t);
}
}
}
var indentStack = new Stack <LineInfo>();
var current = LineInfo.Empty;
while (tokenStack.Count > 0)
{
var t = tokenStack.Pop();
if (t == null)
{
current.NeedsUpdate = true;
continue;
}
var tline = new Lazy <string>(() => snapshot.GetLineFromLineNumber(t.Value.LineNumber).GetText());
if (IsOpenGrouping(t.Value, snapshot))
{
indentStack.Push(current);
var next = tokenStack.Count > 0 ? tokenStack.Peek() : null;
if (next != null && next.Value.LineNumber == t.Value.LineNumber)
{
// Put indent at same depth as grouping
current = new LineInfo {
Indentation = t.Value.ToSourceSpan().End.Column - 1
};
}
else
{
// Put indent at one indent deeper than this line
current = new LineInfo {
Indentation = GetIndentation(tline.Value, tabSize) + tabSize
};
}
}
else if (IsCloseGrouping(t.Value, snapshot))
{
if (indentStack.Count > 0)
{
current = indentStack.Pop();
}
else
{
current.NeedsUpdate = true;
}
}
else if (IsExplicitLineJoin(t.Value, snapshot))
{
while (t != null && tokenStack.Count > 0)
{
t = tokenStack.Pop();
}
if (!t.HasValue)
{
continue;
}
}
else if (current.NeedsUpdate == true)
{
current = new LineInfo {
Indentation = GetIndentation(tline.Value, tabSize)
};
}
if (ShouldDedentAfterKeyword(t.Value, snapshot)) // dedent after some statements
{
current.ShouldDedentAfter = true;
}
if (IsColon(t.Value, snapshot) && // indent after a colon
indentStack.Count == 0) // except in a grouping
{
current.ShouldIndentAfter = true;
// If the colon isn't at the end of the line, cancel it out.
// If the following is a ShouldDedentAfterKeyword, only one dedent will occur.
current.ShouldDedentAfter = (tokenStack.Count != 0 && tokenStack.Peek() != null);
}
}
indentation = current.Indentation +
(current.ShouldIndentAfter ? tabSize : 0) -
(current.ShouldDedentAfter ? tabSize : 0);
return(indentation);
}
internal SnapshotSpan?GetStatementRange()
{
var tokenStack = new Stack <ClassificationSpan>();
bool eol = false, finishLine = false;
// Collect all the tokens until we know we're not in any groupings
foreach (var token in this)
{
if (eol)
{
eol = false;
if (!IsExplicitLineJoin(token))
{
tokenStack.Push(null);
if (finishLine)
{
break;
}
}
}
if (token == null)
{
eol = true;
continue;
}
tokenStack.Push(token);
if (token.ClassificationType.IsOfType(PredefinedClassificationTypeNames.Keyword) &&
PythonKeywords.IsOnlyStatementKeyword(token.Span.GetText()))
{
finishLine = true;
}
}
if (tokenStack.Count == 0)
{
return(null);
}
// Now scan forward through the tokens setting our current statement
// start point.
SnapshotPoint start = new SnapshotPoint(_snapshot, 0);
SnapshotPoint end = start;
bool setStart = true;
int nesting = 0;
foreach (var token in tokenStack)
{
if (setStart && token != null)
{
start = token.Span.Start;
setStart = false;
}
if (token == null)
{
if (nesting == 0)
{
setStart = true;
}
}
else
{
end = token.Span.End;
if (token.IsOpenGrouping())
{
++nesting;
}
else if (token.IsCloseGrouping())
{
--nesting;
}
}
}
// Keep going to find the end of the statement
using (var e = ForwardClassificationSpanEnumerator(Classifier, Span.GetStartPoint(Snapshot))) {
eol = false;
while (e.MoveNext())
{
if (e.Current == null)
{
if (nesting == 0)
{
break;
}
eol = true;
}
else
{
eol = false;
if (setStart)
{
// Final token was EOL, so our start is the first
// token moving forwards
start = e.Current.Span.Start;
setStart = false;
}
end = e.Current.Span.End;
}
}
if (setStart)
{
start = Span.GetStartPoint(Snapshot);
}
if (eol)
{
end = Span.GetEndPoint(Snapshot);
}
}
if (end <= start)
{
// No statement here
return(null);
}
return(new SnapshotSpan(start, end));
}
/// <summary>
/// Gets the range of the expression to the left of our starting span.
/// </summary>
/// <param name="nesting">1 if we have an opening parenthesis for sig completion</param>
/// <param name="paramIndex">The current parameter index.</param>
/// <returns></returns>
public SnapshotSpan?GetExpressionRange(int nesting, out int paramIndex, out SnapshotPoint?sigStart, out string lastKeywordArg, out bool isParameterName, bool forCompletion = true)
{
SnapshotSpan?start = null;
paramIndex = 0;
sigStart = null;
bool nestingChanged = false, lastTokenWasCommaOrOperator = true, lastTokenWasKeywordArgAssignment = false;
int otherNesting = 0;
bool isSigHelp = nesting != 0;
isParameterName = false;
lastKeywordArg = null;
ClassificationSpan lastToken = null;
// Walks backwards over all the lines
var enumerator = ReverseClassificationSpanEnumerator(Classifier, _span.GetSpan(_snapshot).End);
if (enumerator.MoveNext())
{
if (enumerator.Current != null && enumerator.Current.ClassificationType == this.Classifier.Provider.StringLiteral)
{
return(enumerator.Current.Span);
}
lastToken = enumerator.Current;
while (ShouldSkipAsLastToken(lastToken, forCompletion) && enumerator.MoveNext())
{
// skip trailing new line if the user is hovering at the end of the line
if (lastToken == null && (nesting + otherNesting == 0))
{
// new line out of a grouping...
return(_span.GetSpan(_snapshot));
}
lastToken = enumerator.Current;
}
int currentParamAtLastColon = -1; // used to track the current param index at this last colon, before we hit a lambda.
SnapshotSpan?startAtLastToken = null;
// Walk backwards over the tokens in the current line
do
{
var token = enumerator.Current;
if (token == null)
{
// new line
if (nesting != 0 || otherNesting != 0 || (enumerator.MoveNext() && IsExplicitLineJoin(enumerator.Current)))
{
// we're in a grouping, or the previous token is an explicit line join, we'll keep going.
continue;
}
else
{
break;
}
}
var text = token.Span.GetText();
if (text == "(")
{
if (nesting != 0)
{
nesting--;
nestingChanged = true;
if (nesting == 0)
{
if (sigStart == null)
{
sigStart = token.Span.Start;
}
}
}
else
{
if (start == null && !forCompletion)
{
// hovering directly over an open paren, don't provide a tooltip
return(null);
}
// figure out if we're a parameter definition
isParameterName = IsParameterNameOpenParen(enumerator);
break;
}
lastTokenWasCommaOrOperator = true;
lastTokenWasKeywordArgAssignment = false;
}
else if (token.IsOpenGrouping())
{
if (otherNesting != 0)
{
otherNesting--;
}
else
{
if (nesting == 0)
{
if (start == null)
{
return(null);
}
break;
}
paramIndex = 0;
}
nestingChanged = true;
lastTokenWasCommaOrOperator = true;
lastTokenWasKeywordArgAssignment = false;
}
else if (text == ")")
{
nesting++;
nestingChanged = true;
lastTokenWasCommaOrOperator = true;
lastTokenWasKeywordArgAssignment = false;
}
else if (token.IsCloseGrouping())
{
otherNesting++;
nestingChanged = true;
lastTokenWasCommaOrOperator = true;
lastTokenWasKeywordArgAssignment = false;
}
else if (token.ClassificationType == Classifier.Provider.Keyword ||
token.ClassificationType == Classifier.Provider.Operator)
{
lastTokenWasKeywordArgAssignment = false;
if (token.ClassificationType == Classifier.Provider.Keyword && text == "lambda")
{
if (currentParamAtLastColon != -1)
{
paramIndex = currentParamAtLastColon;
currentParamAtLastColon = -1;
}
else
{
// fabcd(lambda a, b, c[PARAMINFO]
// We have to be the 1st param.
paramIndex = 0;
}
}
if (text == ":")
{
startAtLastToken = start;
currentParamAtLastColon = paramIndex;
}
if (nesting == 0 && otherNesting == 0)
{
if (start == null)
{
// http://pytools.codeplex.com/workitem/560
// yield_value = 42
// def f():
// yield<ctrl-space>
// yield <ctrl-space>
//
// If we're next to the keyword, just return the keyword.
// If we're after the keyword, return the span of the text proceeding
// the keyword so we can complete after it.
//
// Also repros with "return <ctrl-space>" or "print <ctrl-space>" both
// of which we weren't reporting completions for before
if (forCompletion)
{
if (token.Span.IntersectsWith(_span.GetSpan(_snapshot)))
{
return(token.Span);
}
else
{
return(_span.GetSpan(_snapshot));
}
}
// hovering directly over a keyword, don't provide a tooltip
return(null);
}
else if ((nestingChanged || forCompletion) && token.ClassificationType == Classifier.Provider.Keyword && (text == "def" || text == "class"))
{
return(null);
}
if (text == "*" || text == "**")
{
if (MoveNextSkipExplicitNewLines(enumerator))
{
if (enumerator.Current.ClassificationType == Classifier.Provider.CommaClassification)
{
isParameterName = IsParameterNameComma(enumerator);
}
else if (enumerator.Current.IsOpenGrouping() && enumerator.Current.Span.GetText() == "(")
{
isParameterName = IsParameterNameOpenParen(enumerator);
}
}
}
break;
}
else if ((token.ClassificationType == Classifier.Provider.Keyword &&
PythonKeywords.IsOnlyStatementKeyword(text)) ||
(token.ClassificationType == Classifier.Provider.Operator && IsAssignmentOperator(text)))
{
if (nesting != 0 && text == "=")
{
// keyword argument allowed in signatures
lastTokenWasKeywordArgAssignment = lastTokenWasCommaOrOperator = true;
}
else if (start == null || (nestingChanged && nesting != 0))
{
return(null);
}
else
{
break;
}
}
else if (token.ClassificationType == Classifier.Provider.Keyword &&
(text == "if" || text == "else"))
{
// if and else can be used in an expression context or a statement context
if (currentParamAtLastColon != -1)
{
start = startAtLastToken;
if (start == null)
{
return(null);
}
break;
}
}
lastTokenWasCommaOrOperator = true;
}
else if (token.ClassificationType == Classifier.Provider.DotClassification)
{
lastTokenWasCommaOrOperator = true;
lastTokenWasKeywordArgAssignment = false;
}
else if (token.ClassificationType == Classifier.Provider.CommaClassification)
{
lastTokenWasCommaOrOperator = true;
lastTokenWasKeywordArgAssignment = false;
if (nesting == 0 && otherNesting == 0)
{
if (start == null && !forCompletion)
{
return(null);
}
isParameterName = IsParameterNameComma(enumerator);
break;
}
else if (nesting == 1 && otherNesting == 0 && sigStart == null)
{
paramIndex++;
}
}
else if (token.ClassificationType == Classifier.Provider.Comment)
{
return(null);
}
else if (!lastTokenWasCommaOrOperator)
{
if (nesting == 0 && otherNesting == 0)
{
break;
}
}
else
{
if (lastTokenWasKeywordArgAssignment &&
token.ClassificationType.IsOfType(PredefinedClassificationTypeNames.Identifier) &&
lastKeywordArg == null)
{
if (paramIndex == 0)
{
lastKeywordArg = text;
}
else
{
lastKeywordArg = "";
}
}
lastTokenWasCommaOrOperator = false;
}
start = token.Span;
} while (enumerator.MoveNext());
}
if (start.HasValue && lastToken != null && (lastToken.Span.End.Position - start.Value.Start.Position) >= 0)
{
return(new SnapshotSpan(
Snapshot,
new Span(
start.Value.Start.Position,
lastToken.Span.End.Position - start.Value.Start.Position
)
));
}
return(_span.GetSpan(_snapshot));
}
