public void KeywordClassification27() { var code = string.Join(Environment.NewLine, PythonKeywords.All(PythonLanguageVersion.V27)); code += "\r\nTrue\r\nFalse"; using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) { foreach (var span in helper.AstClassifierSpans) { var text = span.Span.GetText(); if (string.IsNullOrWhiteSpace(text)) { continue; } // None, True and False are special if (text == "None" || text == "True" || text == "False") { Assert.AreEqual("Python builtin", span.ClassificationType.Classification, text); continue; } Assert.AreEqual("keyword", span.ClassificationType.Classification, text); } } }
internal bool IsInGrouping() { // We assume that groupings are correctly matched and keep a simple // nesting count. int nesting = 0; foreach (var token in this) { if (token == null) { continue; } if (token.IsCloseGrouping()) { nesting++; } else if (token.IsOpenGrouping()) { if (nesting-- == 0) { return(true); } } else if (token.ClassificationType.IsOfType(PredefinedClassificationTypeNames.Keyword) && PythonKeywords.IsOnlyStatementKeyword(token.Span.GetText())) { return(false); } } return(false); }
/// <summary> /// Walks backwards to figure out if we're a parameter name which comes after a comma. /// </summary> private bool IsParameterNameComma(IEnumerator <ClassificationSpan> enumerator) { int groupingLevel = 1; while (MoveNextSkipExplicitNewLines(enumerator)) { if (enumerator.Current.ClassificationType == Classifier.Provider.Keyword) { if (enumerator.Current.Span.GetText() == "def" && groupingLevel == 0) { return(true); } if (PythonKeywords.IsOnlyStatementKeyword(enumerator.Current.Span.GetText())) { return(false); } } if (enumerator.Current.IsOpenGrouping()) { groupingLevel--; if (groupingLevel == 0) { return(IsParameterNameOpenParen(enumerator)); } } else if (enumerator.Current.IsCloseGrouping()) { groupingLevel++; } } return(false); }
internal static bool IsValidPythonIdentifier(string identifier, PythonLanguageVersion pythonVersion) { if (String.IsNullOrEmpty(identifier) || PythonKeywords.IsKeyword(identifier, pythonVersion)) { return(false); } //Python2 identifiers are only certain ASCII characters if (pythonVersion < PythonLanguageVersion.V30) { return(Python2IdentifierRegex.IsMatch(identifier)); } //Python3 identifiers can include unicode characters if (!Tokenizer.IsIdentifierStartChar(identifier[0])) { return(false); } return(identifier.Skip(1).All(Tokenizer.IsIdentifierChar)); }
internal static bool IsInGrouping(ITextSnapshot snapshot, IEnumerable <TrackingTokenInfo> tokensInReverse) { int nesting = 0; foreach (var token in tokensInReverse) { if (token.Category == Parsing.TokenCategory.Grouping) { var t = token.GetText(snapshot); if (t.IsCloseGrouping()) { nesting++; } else if (t.IsOpenGrouping()) { if (nesting-- == 0) { return(true); } } } else if (token.Category == Parsing.TokenCategory.Delimiter) { if (nesting == 0 && token.GetText(snapshot) == ",") { return(true); } } else if (token.Category == Parsing.TokenCategory.Keyword) { if (PythonKeywords.IsOnlyStatementKeyword(token.GetText(snapshot))) { return(false); } } } return(false); }
public void KeywordClassification33() { var code = string.Join(Environment.NewLine, PythonKeywords.All(PythonLanguageVersion.V33)); using (var helper = new ClassifierHelper(MockTextBuffer(code), PythonLanguageVersion.V33)) { foreach (var span in helper.AstClassifierSpans) { var text = span.Span.GetText(); if (string.IsNullOrWhiteSpace(text)) { continue; } // None is special if (text == "None") { Assert.AreEqual("Python builtin", span.ClassificationType.Classification, text); continue; } Assert.AreEqual("keyword", span.ClassificationType.Classification, text); } } }
private static int CalculateIndentation(string baseline, ITextSnapshotLine line, IEditorOptions options, IClassifier classifier, ITextView textView) { int indentation = GetIndentation(baseline, options.GetTabSize()); int tabSize = options.GetIndentSize(); var tokens = classifier.GetClassificationSpans(line.Extent); if (tokens.Count > 0 && !IsUnterminatedStringToken(tokens[tokens.Count - 1])) { int tokenIndex = tokens.Count - 1; while (tokenIndex >= 0 && (tokens[tokenIndex].ClassificationType.IsOfType(PredefinedClassificationTypeNames.Comment) || tokens[tokenIndex].ClassificationType.IsOfType(PredefinedClassificationTypeNames.WhiteSpace))) { tokenIndex--; } if (tokenIndex < 0) { return(indentation); } if (ReverseExpressionParser.IsExplicitLineJoin(tokens[tokenIndex])) { // explicit line continuation, we indent 1 level for the continued line unless // we're already indented because of multiple line continuation characters. indentation = GetIndentation(line.GetText(), options.GetTabSize()); var joinedLine = tokens[tokenIndex].Span.Start.GetContainingLine(); if (joinedLine.LineNumber > 0) { var prevLineSpans = classifier.GetClassificationSpans(tokens[tokenIndex].Span.Snapshot.GetLineFromLineNumber(joinedLine.LineNumber - 1).Extent); if (prevLineSpans.Count == 0 || !ReverseExpressionParser.IsExplicitLineJoin(prevLineSpans[prevLineSpans.Count - 1])) { indentation += tabSize; } } else { indentation += tabSize; } return(indentation); } string sline = tokens[tokenIndex].Span.GetText(); var lastChar = sline.Length == 0 ? '\0' : sline[sline.Length - 1]; // use the expression parser to figure out if we're in a grouping... var spans = textView.BufferGraph.MapDownToFirstMatch( tokens[tokenIndex].Span, SpanTrackingMode.EdgePositive, PythonContentTypePrediciate ); if (spans.Count == 0) { return(indentation); } var revParser = new ReverseExpressionParser( spans[0].Snapshot, spans[0].Snapshot.TextBuffer, spans[0].Snapshot.CreateTrackingSpan( spans[0].Span, SpanTrackingMode.EdgePositive ) ); var tokenStack = new System.Collections.Generic.Stack <ClassificationSpan>(); tokenStack.Push(null); // end with an implicit newline bool endAtNextNull = false; foreach (var token in revParser) { tokenStack.Push(token); if (token == null && endAtNextNull) { break; } else if (token != null && token.ClassificationType == revParser.Classifier.Provider.Keyword && PythonKeywords.IsOnlyStatementKeyword(token.Span.GetText())) { endAtNextNull = true; } } var indentStack = new System.Collections.Generic.Stack <LineInfo>(); var current = LineInfo.Empty; while (tokenStack.Count > 0) { var token = tokenStack.Pop(); if (token == null) { current.NeedsUpdate = true; } else if (token.IsOpenGrouping()) { indentStack.Push(current); var start = token.Span.Start; var line2 = start.GetContainingLine(); var next = tokenStack.Count > 0 ? tokenStack.Peek() : null; if (next != null && next.Span.End <= line2.End) { current = new LineInfo { Indentation = start.Position - line2.Start.Position + 1 }; } else { current = new LineInfo { Indentation = GetIndentation(line2.GetText(), tabSize) + tabSize }; } } else if (token.IsCloseGrouping()) { if (indentStack.Count > 0) { current = indentStack.Pop(); } else { current.NeedsUpdate = true; } } else if (ReverseExpressionParser.IsExplicitLineJoin(token)) { while (token != null && tokenStack.Count > 0) { token = tokenStack.Pop(); } } else if (current.NeedsUpdate == true) { var line2 = token.Span.Start.GetContainingLine(); current = new LineInfo { Indentation = GetIndentation(line2.GetText(), tabSize) }; } if (token != null && ShouldDedentAfterKeyword(token)) // dedent after some statements { current.ShouldDedentAfter = true; } if (token != null && token.Span.GetText() == ":" && // indent after a colon indentStack.Count == 0) // except in a grouping { current.ShouldIndentAfter = true; // If the colon isn't at the end of the line, cancel it out. // If the following is a ShouldDedentAfterKeyword, only one dedent will occur. current.ShouldDedentAfter = (tokenStack.Count != 0 && tokenStack.Peek() != null); } } indentation = current.Indentation + (current.ShouldIndentAfter ? tabSize : 0) - (current.ShouldDedentAfter ? tabSize : 0); } // Map indentation back to the view's text buffer. int offset = 0; var viewLineStart = textView.BufferGraph.MapUpToSnapshot(line.Start, PointTrackingMode.Positive, PositionAffinity.Successor, textView.TextSnapshot); if (viewLineStart.HasValue) { offset = viewLineStart.Value.Position - viewLineStart.Value.GetContainingLine().Start.Position; } return(offset + indentation); }
private static int CalculateIndentation( string baseline, ITextSnapshotLine line, IEditorOptions options, PythonTextBufferInfo buffer ) { var snapshot = line.Snapshot; if (snapshot.TextBuffer != buffer.Buffer) { throw new ArgumentException("buffer mismatch"); } int indentation = GetIndentation(baseline, options.GetTabSize()); int tabSize = options.GetIndentSize(); var tokens = buffer.GetTokens(line).ToList(); while (tokens.Count > 0 && IsWhitespace(tokens[tokens.Count - 1].Category)) { tokens.RemoveAt(tokens.Count - 1); } if (tokens.Count == 0 || IsUnterminatedStringToken(tokens[tokens.Count - 1], snapshot)) { return(indentation); } if (HasExplicitLineJoin(tokens, snapshot)) { // explicit line continuation, we indent 1 level for the continued line unless // we're already indented because of multiple line continuation characters. indentation = GetIndentation(line.GetText(), options.GetTabSize()); var joinedLine = line.LineNumber - 1; if (joinedLine >= 0) { var prevLineTokens = buffer.GetTokens(snapshot.GetLineFromLineNumber(joinedLine)).ToList(); if (prevLineTokens.Count == 0 || !HasExplicitLineJoin(prevLineTokens, snapshot)) { indentation += tabSize; } } else { indentation += tabSize; } return(indentation); } var tokenStack = new Stack <TrackingTokenInfo?>(); tokenStack.Push(null); // end with an implicit newline int endAtLine = -1, currentLine = tokens.Last().LineNumber; foreach (var t in buffer.GetTokensInReverseFromPoint(tokens.Last().ToSnapshotSpan(snapshot).Start)) { if (t.LineNumber == currentLine) { tokenStack.Push(t); } else { tokenStack.Push(null); } if (t.LineNumber == endAtLine) { break; } else if (t.Category == TokenCategory.Keyword && PythonKeywords.IsOnlyStatementKeyword(t.GetText(snapshot), buffer.LanguageVersion)) { endAtLine = t.LineNumber - 1; } if (t.LineNumber != currentLine) { currentLine = t.LineNumber; if (t.Category != TokenCategory.WhiteSpace && t.Category != TokenCategory.Comment && t.Category != TokenCategory.LineComment) { tokenStack.Push(t); } } } var indentStack = new Stack <LineInfo>(); var current = LineInfo.Empty; while (tokenStack.Count > 0) { var t = tokenStack.Pop(); if (t == null) { current.NeedsUpdate = true; continue; } var tline = new Lazy <string>(() => snapshot.GetLineFromLineNumber(t.Value.LineNumber).GetText()); if (IsOpenGrouping(t.Value, snapshot)) { indentStack.Push(current); var next = tokenStack.Count > 0 ? tokenStack.Peek() : null; if (next != null && next.Value.LineNumber == t.Value.LineNumber) { // Put indent at same depth as grouping current = new LineInfo { Indentation = t.Value.ToSourceSpan().End.Column - 1 }; } else { // Put indent at one indent deeper than this line current = new LineInfo { Indentation = GetIndentation(tline.Value, tabSize) + tabSize }; } } else if (IsCloseGrouping(t.Value, snapshot)) { if (indentStack.Count > 0) { current = indentStack.Pop(); } else { current.NeedsUpdate = true; } } else if (IsExplicitLineJoin(t.Value, snapshot)) { while (t != null && tokenStack.Count > 0) { t = tokenStack.Pop(); } if (!t.HasValue) { continue; } } else if (current.NeedsUpdate == true) { current = new LineInfo { Indentation = GetIndentation(tline.Value, tabSize) }; } if (ShouldDedentAfterKeyword(t.Value, snapshot)) // dedent after some statements { current.ShouldDedentAfter = true; } if (IsColon(t.Value, snapshot) && // indent after a colon indentStack.Count == 0) // except in a grouping { current.ShouldIndentAfter = true; // If the colon isn't at the end of the line, cancel it out. // If the following is a ShouldDedentAfterKeyword, only one dedent will occur. current.ShouldDedentAfter = (tokenStack.Count != 0 && tokenStack.Peek() != null); } } indentation = current.Indentation + (current.ShouldIndentAfter ? tabSize : 0) - (current.ShouldDedentAfter ? tabSize : 0); return(indentation); }
internal SnapshotSpan?GetStatementRange() { var tokenStack = new Stack <ClassificationSpan>(); bool eol = false, finishLine = false; // Collect all the tokens until we know we're not in any groupings foreach (var token in this) { if (eol) { eol = false; if (!IsExplicitLineJoin(token)) { tokenStack.Push(null); if (finishLine) { break; } } } if (token == null) { eol = true; continue; } tokenStack.Push(token); if (token.ClassificationType.IsOfType(PredefinedClassificationTypeNames.Keyword) && PythonKeywords.IsOnlyStatementKeyword(token.Span.GetText())) { finishLine = true; } } if (tokenStack.Count == 0) { return(null); } // Now scan forward through the tokens setting our current statement // start point. SnapshotPoint start = new SnapshotPoint(_snapshot, 0); SnapshotPoint end = start; bool setStart = true; int nesting = 0; foreach (var token in tokenStack) { if (setStart && token != null) { start = token.Span.Start; setStart = false; } if (token == null) { if (nesting == 0) { setStart = true; } } else { end = token.Span.End; if (token.IsOpenGrouping()) { ++nesting; } else if (token.IsCloseGrouping()) { --nesting; } } } // Keep going to find the end of the statement using (var e = ForwardClassificationSpanEnumerator(Classifier, Span.GetStartPoint(Snapshot))) { eol = false; while (e.MoveNext()) { if (e.Current == null) { if (nesting == 0) { break; } eol = true; } else { eol = false; if (setStart) { // Final token was EOL, so our start is the first // token moving forwards start = e.Current.Span.Start; setStart = false; } end = e.Current.Span.End; } } if (setStart) { start = Span.GetStartPoint(Snapshot); } if (eol) { end = Span.GetEndPoint(Snapshot); } } if (end <= start) { // No statement here return(null); } return(new SnapshotSpan(start, end)); }
/// <summary> /// Gets the range of the expression to the left of our starting span. /// </summary> /// <param name="nesting">1 if we have an opening parenthesis for sig completion</param> /// <param name="paramIndex">The current parameter index.</param> /// <returns></returns> public SnapshotSpan?GetExpressionRange(int nesting, out int paramIndex, out SnapshotPoint?sigStart, out string lastKeywordArg, out bool isParameterName, bool forCompletion = true) { SnapshotSpan?start = null; paramIndex = 0; sigStart = null; bool nestingChanged = false, lastTokenWasCommaOrOperator = true, lastTokenWasKeywordArgAssignment = false; int otherNesting = 0; bool isSigHelp = nesting != 0; isParameterName = false; lastKeywordArg = null; ClassificationSpan lastToken = null; // Walks backwards over all the lines var enumerator = ReverseClassificationSpanEnumerator(Classifier, _span.GetSpan(_snapshot).End); if (enumerator.MoveNext()) { if (enumerator.Current != null && enumerator.Current.ClassificationType == this.Classifier.Provider.StringLiteral) { return(enumerator.Current.Span); } lastToken = enumerator.Current; while (ShouldSkipAsLastToken(lastToken, forCompletion) && enumerator.MoveNext()) { // skip trailing new line if the user is hovering at the end of the line if (lastToken == null && (nesting + otherNesting == 0)) { // new line out of a grouping... return(_span.GetSpan(_snapshot)); } lastToken = enumerator.Current; } int currentParamAtLastColon = -1; // used to track the current param index at this last colon, before we hit a lambda. SnapshotSpan?startAtLastToken = null; // Walk backwards over the tokens in the current line do { var token = enumerator.Current; if (token == null) { // new line if (nesting != 0 || otherNesting != 0 || (enumerator.MoveNext() && IsExplicitLineJoin(enumerator.Current))) { // we're in a grouping, or the previous token is an explicit line join, we'll keep going. continue; } else { break; } } var text = token.Span.GetText(); if (text == "(") { if (nesting != 0) { nesting--; nestingChanged = true; if (nesting == 0) { if (sigStart == null) { sigStart = token.Span.Start; } } } else { if (start == null && !forCompletion) { // hovering directly over an open paren, don't provide a tooltip return(null); } // figure out if we're a parameter definition isParameterName = IsParameterNameOpenParen(enumerator); break; } lastTokenWasCommaOrOperator = true; lastTokenWasKeywordArgAssignment = false; } else if (token.IsOpenGrouping()) { if (otherNesting != 0) { otherNesting--; } else { if (nesting == 0) { if (start == null) { return(null); } break; } paramIndex = 0; } nestingChanged = true; lastTokenWasCommaOrOperator = true; lastTokenWasKeywordArgAssignment = false; } else if (text == ")") { nesting++; nestingChanged = true; lastTokenWasCommaOrOperator = true; lastTokenWasKeywordArgAssignment = false; } else if (token.IsCloseGrouping()) { otherNesting++; nestingChanged = true; lastTokenWasCommaOrOperator = true; lastTokenWasKeywordArgAssignment = false; } else if (token.ClassificationType == Classifier.Provider.Keyword || token.ClassificationType == Classifier.Provider.Operator) { lastTokenWasKeywordArgAssignment = false; if (token.ClassificationType == Classifier.Provider.Keyword && text == "lambda") { if (currentParamAtLastColon != -1) { paramIndex = currentParamAtLastColon; currentParamAtLastColon = -1; } else { // fabcd(lambda a, b, c[PARAMINFO] // We have to be the 1st param. paramIndex = 0; } } if (text == ":") { startAtLastToken = start; currentParamAtLastColon = paramIndex; } if (nesting == 0 && otherNesting == 0) { if (start == null) { // http://pytools.codeplex.com/workitem/560 // yield_value = 42 // def f(): // yield<ctrl-space> // yield <ctrl-space> // // If we're next to the keyword, just return the keyword. // If we're after the keyword, return the span of the text proceeding // the keyword so we can complete after it. // // Also repros with "return <ctrl-space>" or "print <ctrl-space>" both // of which we weren't reporting completions for before if (forCompletion) { if (token.Span.IntersectsWith(_span.GetSpan(_snapshot))) { return(token.Span); } else { return(_span.GetSpan(_snapshot)); } } // hovering directly over a keyword, don't provide a tooltip return(null); } else if ((nestingChanged || forCompletion) && token.ClassificationType == Classifier.Provider.Keyword && (text == "def" || text == "class")) { return(null); } if (text == "*" || text == "**") { if (MoveNextSkipExplicitNewLines(enumerator)) { if (enumerator.Current.ClassificationType == Classifier.Provider.CommaClassification) { isParameterName = IsParameterNameComma(enumerator); } else if (enumerator.Current.IsOpenGrouping() && enumerator.Current.Span.GetText() == "(") { isParameterName = IsParameterNameOpenParen(enumerator); } } } break; } else if ((token.ClassificationType == Classifier.Provider.Keyword && PythonKeywords.IsOnlyStatementKeyword(text)) || (token.ClassificationType == Classifier.Provider.Operator && IsAssignmentOperator(text))) { if (nesting != 0 && text == "=") { // keyword argument allowed in signatures lastTokenWasKeywordArgAssignment = lastTokenWasCommaOrOperator = true; } else if (start == null || (nestingChanged && nesting != 0)) { return(null); } else { break; } } else if (token.ClassificationType == Classifier.Provider.Keyword && (text == "if" || text == "else")) { // if and else can be used in an expression context or a statement context if (currentParamAtLastColon != -1) { start = startAtLastToken; if (start == null) { return(null); } break; } } lastTokenWasCommaOrOperator = true; } else if (token.ClassificationType == Classifier.Provider.DotClassification) { lastTokenWasCommaOrOperator = true; lastTokenWasKeywordArgAssignment = false; } else if (token.ClassificationType == Classifier.Provider.CommaClassification) { lastTokenWasCommaOrOperator = true; lastTokenWasKeywordArgAssignment = false; if (nesting == 0 && otherNesting == 0) { if (start == null && !forCompletion) { return(null); } isParameterName = IsParameterNameComma(enumerator); break; } else if (nesting == 1 && otherNesting == 0 && sigStart == null) { paramIndex++; } } else if (token.ClassificationType == Classifier.Provider.Comment) { return(null); } else if (!lastTokenWasCommaOrOperator) { if (nesting == 0 && otherNesting == 0) { break; } } else { if (lastTokenWasKeywordArgAssignment && token.ClassificationType.IsOfType(PredefinedClassificationTypeNames.Identifier) && lastKeywordArg == null) { if (paramIndex == 0) { lastKeywordArg = text; } else { lastKeywordArg = ""; } } lastTokenWasCommaOrOperator = false; } start = token.Span; } while (enumerator.MoveNext()); } if (start.HasValue && lastToken != null && (lastToken.Span.End.Position - start.Value.Start.Position) >= 0) { return(new SnapshotSpan( Snapshot, new Span( start.Value.Start.Position, lastToken.Span.End.Position - start.Value.Start.Position ) )); } return(_span.GetSpan(_snapshot)); }