public void AsyncAwaitClassification()
{
var code = @"
await f
await + f
async with f: pass
async for x in f: pass
async def f():
await f
async with f: pass
async for x in f: pass
class F:
async def f(self): [x async for x in y]
@F
async def f():
pass
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V36)) {
helper.CheckAstClassifierSpans("ii i+i iki:k ikiki:k iki(): ii iki:k ikiki:k ki: " +
"iki(i): (iikiki) @iiki():k");
// "await f" does not highlight "f", but "await + f" does
// also note that only async and await are keywords here - not 'for'
helper.CheckAnalysisClassifierSpans("fff k<async>f k<await>f k<async>f k<async>f c<F> " +
"k<async>fp<self>p<x>k<async>p c<F>k<async>f");
}
}
public void KeywordClassification27()
{
var code = string.Join(Environment.NewLine, PythonKeywords.All(PythonLanguageVersion.V27));
code += "\r\nTrue\r\nFalse";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
foreach (var span in helper.AstClassifierSpans)
{
var text = span.Span.GetText();
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
// None, True and False are special
if (text == "None" || text == "True" || text == "False")
{
Assert.AreEqual("Python builtin", span.ClassificationType.Classification, text);
continue;
}
Assert.AreEqual("keyword", span.ClassificationType.Classification, text);
}
}
}
public void AsyncAwaitClassification()
{
var code = @"
await f
await + f
async with f: pass
async for x in f: pass
async def f():
await f
async with f: pass
async for x in f: pass
class F:
async def f(self): pass
@F
async def f():
pass
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V35)) {
helper.CheckAstClassifierSpans("ii i+i iki:k ikiki:k iki(): ii iki:k ikiki:k ki: iki(i): k @iiki():k");
// "await f" does not highlight "f", but "await + f" does
helper.CheckAnalysisClassifierSpans("fff k<async>f k<await>f k<async>f k<async>f c<F> k<async>fp c<F>k<async>f");
}
}
public void ReturnAnnotationClassifications()
{
var code = @"
def f() -> int:
pass
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V35)) {
helper.CheckAnalysisClassifierSpans("f<f>c<int>");
}
}
public void MultilineStringClassification()
{
var code = @"t = '''a
b ''' + c + 'x' + '''d
e'''";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("i=sss+i+s+sss");
}
}
public void TrueFalseClassification()
{
var code = "True False";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("b<True> b<False>");
}
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V33)) {
helper.CheckAstClassifierSpans("k<True> k<False>");
}
}
public void DocStringClassifications()
{
var code = @"def f():
'''doc string'''
'''not a doc string'''
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V35)) {
helper.CheckAstClassifierSpans("ki():ss");
helper.CheckAnalysisClassifierSpans("f<f>d<'''doc string'''>");
}
}
public void ParameterAnnotationClassification()
{
var code = @"class A: pass
class B: pass
def f(a = A, b : B):
pass
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("ki:k ki:k ki(i=i,i:i): k");
helper.CheckAnalysisClassifierSpans("c<A>c<B>f<f>pc<A>pc<B>");
}
}
public void ImportClassifications()
{
var code = @"import abc as x
from os import fdopen
abc
x
os
fdopen
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("kiki kiki i i i i");
helper.CheckAnalysisClassifierSpans("m<abc>m<x>m<os>m<x>");
}
}
public void TypeClassification()
{
var code = @"class MyClass(object):
pass
mc = MyClass()
MyClassAlias = MyClass
mca = MyClassAlias()
MyClassType = type(mc)
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("ki(i): k i=i() i=i i=i() i=i(i)");
helper.CheckAnalysisClassifierSpans("c<MyClass>c<object>cc<MyClassAlias>ccc<type>");
}
}
public void ModuleClassification()
{
var code = @"import abc
import os
import ntpath
os.path = ntpath
abc = 123
abc = True
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("ki ki ki i.i=i i=n i=b");
helper.CheckAnalysisClassifierSpans("m<abc>m<os>m<ntpath>m<os>m<ntpath>m<abc>m<abc>");
}
}
public void ParameterClassification()
{
var code = @"def f(a, b, c):
a = b
b = c
return a
f(a, b, c)
a = b
b = c
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("ki(i,i,i): i=i i=i ki i(i,i,i) i=i i=i");
helper.CheckAnalysisClassifierSpans("f<f>ppppppppf<f>");
}
}
private static async Task <ClassifiedSpan[]> GetClassifiedSpansForDocumentAsync(
Document document,
TextSpan textSpan,
ClassificationOptions options,
bool includeSyntacticClassifications,
CancellationToken cancellationToken)
{
var classificationService = document.GetRequiredLanguageService <IClassificationService>();
using var _ = ArrayBuilder <ClassifiedSpan> .GetInstance(out var classifiedSpans);
// Case 1 - Generated Razor documents:
// In Razor, the C# syntax classifier does not run on the client. This means we need to return both
// syntactic and semantic classifications.
// Case 2 - C# and VB documents:
// In C#/VB, the syntax classifier runs on the client. This means we only need to return semantic
// classifications.
//
// Ideally, Razor will eventually run the classifier on their end so we can get rid of this special
// casing: https://github.com/dotnet/razor-tooling/issues/5850
if (includeSyntacticClassifications)
{
// `removeAdditiveSpans` will remove token modifiers such as 'static', which we want to include in LSP.
// `fillInClassifiedSpanGaps` includes whitespace in the results, which we don't care about in LSP.
// Therefore, we set both optional parameters to false.
var spans = await ClassifierHelper.GetClassifiedSpansAsync(
document, textSpan, options, cancellationToken, removeAdditiveSpans : false, fillInClassifiedSpanGaps : false).ConfigureAwait(false);
// The spans returned to us may include some empty spans, which we don't care about.
var nonEmptySpans = spans.Where(s => !s.TextSpan.IsEmpty);
classifiedSpans.AddRange(nonEmptySpans);
}
else
{
await classificationService.AddSemanticClassificationsAsync(
document, textSpan, options, classifiedSpans, cancellationToken).ConfigureAwait(false);
await classificationService.AddEmbeddedLanguageClassificationsAsync(
document, textSpan, options, classifiedSpans, cancellationToken).ConfigureAwait(false);
}
// Classified spans are not guaranteed to be returned in a certain order so we sort them to be safe.
classifiedSpans.Sort(ClassifiedSpanComparer.Instance);
return(classifiedSpans.ToArray());
}
public void ImportClassifications()
{
// We import a name that is not in os, because otherwise
// its classification may depend on the current DB for the
// module.
var code = @"import abc as x
from os import name_not_in_os
abc
x
os
name_not_in_os
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("kiki kiki i i i i");
helper.CheckAnalysisClassifierSpans("m<abc>m<x>m<os>m<x>");
}
}
public void KeywordClassification33() {
var code = string.Join(Environment.NewLine, PythonKeywords.All(PythonLanguageVersion.V33));
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V33)) {
foreach (var span in helper.AstClassifierSpans) {
var text = span.Span.GetText();
if (string.IsNullOrWhiteSpace(text)) {
continue;
}
// None is special
if (text == "None") {
Assert.AreEqual("Python builtin", span.ClassificationType.Classification, text);
continue;
}
Assert.AreEqual("keyword", span.ClassificationType.Classification, text);
}
}
}
public void RegexClassifications()
{
var code = @"import re as R
R.compile('pattern', 'str')
R.escape('pattern', 'str')
R.findall('pattern', 'str')
R.finditer('pattern', 'str')
R.fullmatch('pattern', 'str')
R.match('pattern', 'str')
R.search('pattern', 'str')
R.split('pattern', 'str')
R.sub('pattern', 'str')
R.subn('pattern', 'str')
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V35)) {
helper.CheckAstClassifierSpans("kiki " + string.Join(" ", Enumerable.Repeat("i.i(s,s)", 10)));
helper.CheckAnalysisClassifierSpans("m<re>m<R> " + string.Join(" ", Enumerable.Repeat("m<R>fr", 10)));
}
}
public void KeywordClassification33()
{
var code = string.Join(Environment.NewLine, PythonKeywords.All(PythonLanguageVersion.V33));
using (var helper = new ClassifierHelper(MockTextBuffer(code), PythonLanguageVersion.V33)) {
foreach (var span in helper.AstClassifierSpans)
{
var text = span.Span.GetText();
if (string.IsNullOrWhiteSpace(text))
{
continue;
}
// None is special
if (text == "None")
{
Assert.AreEqual("Python builtin", span.ClassificationType.Classification, text);
continue;
}
Assert.AreEqual("keyword", span.ClassificationType.Classification, text);
}
}
}
public void ParameterAnnotationClassification() {
var code = @"class A: pass
class B: pass
def f(a = A, b : B):
pass
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("ki:k ki:k ki(i=i,i:i): k");
helper.CheckAnalysisClassifierSpans("c<A>c<B>f<f>pc<A>pc<B>");
}
}
public void ImportClassifications() {
var code = @"import abc as x
from os import fdopen
abc
x
os
fdopen
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("kiki kiki i i i i");
helper.CheckAnalysisClassifierSpans("m<abc>m<x>m<os>m<x>");
}
}
public void AsyncAwaitClassification() {
var code = @"
await f
await + f
async with f: pass
async for x in f: pass
async def f():
await f
async with f: pass
async for x in f: pass
class F:
async def f(self): pass
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V35)) {
helper.CheckAstClassifierSpans("ii i+i iki:k ikiki:k iki(): ii iki:k ikiki:k ki: iki(i): k");
// "await f" does not highlight "f", but "await + f" does
helper.CheckAnalysisClassifierSpans("fff k<async>f k<await>f k<async>f k<async>f c<F> k<async>fp");
}
}
public void ModuleClassification() {
var code = @"import abc
import os
import ntpath
os.path = ntpath
abc = 123
abc = True
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("ki ki ki i.i=i i=n i=b");
helper.CheckAnalysisClassifierSpans("m<abc>m<os>m<ntpath>m<os>m<ntpath>m<abc>m<abc>");
}
}
public void ParameterClassification() {
var code = @"def f(a, b, c):
a = b
b = c
return a
f(a, b, c)
a = b
b = c
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("ki(i,i,i): i=i i=i ki i(i,i,i) i=i i=i");
helper.CheckAnalysisClassifierSpans("f<f>ppppppppf<f>");
}
}
private static async Task <ContainerElement> BuildInteractiveContentAsync(CodeAnalysisQuickInfoItem quickInfoItem,
IntellisenseQuickInfoBuilderContext?context,
CancellationToken cancellationToken)
{
// Build the first line of QuickInfo item, the images and the Description section should be on the first line with Wrapped style
var glyphs = quickInfoItem.Tags.GetGlyphs();
var symbolGlyph = glyphs.FirstOrDefault(g => g != Glyph.CompletionWarning);
var warningGlyph = glyphs.FirstOrDefault(g => g == Glyph.CompletionWarning);
var firstLineElements = new List <object>();
if (symbolGlyph != Glyph.None)
{
firstLineElements.Add(new ImageElement(symbolGlyph.GetImageId()));
}
if (warningGlyph != Glyph.None)
{
firstLineElements.Add(new ImageElement(warningGlyph.GetImageId()));
}
var elements = new List <object>();
var descSection = quickInfoItem.Sections.FirstOrDefault(s => s.Kind == QuickInfoSectionKinds.Description);
if (descSection != null)
{
var isFirstElement = true;
foreach (var element in Helpers.BuildInteractiveTextElements(descSection.TaggedParts, context))
{
if (isFirstElement)
{
isFirstElement = false;
firstLineElements.Add(element);
}
else
{
// If the description section contains multiple paragraphs, the second and additional paragraphs
// are not wrapped in firstLineElements (they are normal paragraphs).
elements.Add(element);
}
}
}
elements.Insert(0, new ContainerElement(ContainerElementStyle.Wrapped, firstLineElements));
var documentationCommentSection = quickInfoItem.Sections.FirstOrDefault(s => s.Kind == QuickInfoSectionKinds.DocumentationComments);
if (documentationCommentSection != null)
{
var isFirstElement = true;
foreach (var element in Helpers.BuildInteractiveTextElements(documentationCommentSection.TaggedParts, context))
{
if (isFirstElement)
{
isFirstElement = false;
// Stack the first paragraph of the documentation comments with the last line of the description
// to avoid vertical padding between the two.
var lastElement = elements[elements.Count - 1];
elements[elements.Count - 1] = new ContainerElement(
ContainerElementStyle.Stacked,
lastElement,
element);
}
else
{
elements.Add(element);
}
}
}
// Add the remaining sections as Stacked style
elements.AddRange(
quickInfoItem.Sections.Where(s => s.Kind is not QuickInfoSectionKinds.Description and not QuickInfoSectionKinds.DocumentationComments)
.SelectMany(s => Helpers.BuildInteractiveTextElements(s.TaggedParts, context)));
// build text for RelatedSpan
if (quickInfoItem.RelatedSpans.Any() && context?.Document is Document document)
{
var textRuns = new List <ClassifiedTextRun>();
var spanSeparatorNeededBefore = false;
foreach (var span in quickInfoItem.RelatedSpans)
{
var classifiedSpans = await ClassifierHelper.GetClassifiedSpansAsync(document, span, cancellationToken).ConfigureAwait(false);
var tabSize = document.Project.Solution.Options.GetOption(FormattingOptions.TabSize, document.Project.Language);
var text = await document.GetTextAsync(cancellationToken).ConfigureAwait(false);
var spans = IndentationHelper.GetSpansWithAlignedIndentation(text, classifiedSpans.ToImmutableArray(), tabSize);
var textRunsOfSpan = spans.Select(s => new ClassifiedTextRun(s.ClassificationType, text.GetSubText(s.TextSpan).ToString(), ClassifiedTextRunStyle.UseClassificationFont)).ToList();
if (textRunsOfSpan.Count > 0)
{
if (spanSeparatorNeededBefore)
{
textRuns.Add(new ClassifiedTextRun(ClassificationTypeNames.WhiteSpace, "\r\n", ClassifiedTextRunStyle.UseClassificationFont));
}
textRuns.AddRange(textRunsOfSpan);
spanSeparatorNeededBefore = true;
}
}
if (textRuns.Any())
{
elements.Add(new ClassifiedTextElement(textRuns));
}
}
return(new ContainerElement(
ContainerElementStyle.Stacked | ContainerElementStyle.VerticalPadding,
elements));
}
public void TypeClassification() {
var code = @"class MyClass(object):
pass
mc = MyClass()
MyClassAlias = MyClass
mca = MyClassAlias()
MyClassType = type(mc)
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("ki(i): k i=i() i=i i=i() i=i(i)");
helper.CheckAnalysisClassifierSpans("c<MyClass>c<object>cc<MyClassAlias>ccc<type>");
}
}
public void TrueFalseClassification() {
var code = "True False";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V27)) {
helper.CheckAstClassifierSpans("b<True> b<False>");
}
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V33)) {
helper.CheckAstClassifierSpans("k<True> k<False>");
}
}
internal static async Task <IntellisenseQuickInfoItem> BuildItemAsync(ITrackingSpan trackingSpan,
CodeAnalysisQuickInfoItem quickInfoItem,
ITextSnapshot snapshot,
Document document,
Lazy <IStreamingFindUsagesPresenter> streamingPresenter,
CancellationToken cancellationToken)
{
// Build the first line of QuickInfo item, the images and the Description section should be on the first line with Wrapped style
var glyphs = quickInfoItem.Tags.GetGlyphs();
var symbolGlyph = glyphs.FirstOrDefault(g => g != Glyph.CompletionWarning);
var warningGlyph = glyphs.FirstOrDefault(g => g == Glyph.CompletionWarning);
var firstLineElements = new List <object>();
if (symbolGlyph != Glyph.None)
{
firstLineElements.Add(new ImageElement(symbolGlyph.GetImageId()));
}
if (warningGlyph != Glyph.None)
{
firstLineElements.Add(new ImageElement(warningGlyph.GetImageId()));
}
var elements = new List <object>();
var descSection = quickInfoItem.Sections.FirstOrDefault(s => s.Kind == QuickInfoSectionKinds.Description);
if (descSection != null)
{
var isFirstElement = true;
foreach (var element in Helpers.BuildInteractiveTextElements(descSection.TaggedParts, document, streamingPresenter))
{
if (isFirstElement)
{
isFirstElement = false;
firstLineElements.Add(element);
}
else
{
// If the description section contains multiple paragraphs, the second and additional paragraphs
// are not wrapped in firstLineElements (they are normal paragraphs).
elements.Add(element);
}
}
}
elements.Insert(0, new ContainerElement(ContainerElementStyle.Wrapped, firstLineElements));
var documentationCommentSection = quickInfoItem.Sections.FirstOrDefault(s => s.Kind == QuickInfoSectionKinds.DocumentationComments);
if (documentationCommentSection != null)
{
var isFirstElement = true;
foreach (var element in Helpers.BuildInteractiveTextElements(documentationCommentSection.TaggedParts, document, streamingPresenter))
{
if (isFirstElement)
{
isFirstElement = false;
// Stack the first paragraph of the documentation comments with the last line of the description
// to avoid vertical padding between the two.
var lastElement = elements[elements.Count - 1];
elements[elements.Count - 1] = new ContainerElement(
ContainerElementStyle.Stacked,
lastElement,
element);
}
else
{
elements.Add(element);
}
}
}
// Add the remaining sections as Stacked style
elements.AddRange(
quickInfoItem.Sections.Where(s => s.Kind != QuickInfoSectionKinds.Description && s.Kind != QuickInfoSectionKinds.DocumentationComments)
.SelectMany(s => Helpers.BuildInteractiveTextElements(s.TaggedParts, document, streamingPresenter)));
// build text for RelatedSpan
if (quickInfoItem.RelatedSpans.Any())
{
var classifiedSpanList = new List <ClassifiedSpan>();
foreach (var span in quickInfoItem.RelatedSpans)
{
var classifiedSpans = await ClassifierHelper.GetClassifiedSpansAsync(document, span, cancellationToken).ConfigureAwait(false);
classifiedSpanList.AddRange(classifiedSpans);
}
var tabSize = document.Project.Solution.Workspace.Options.GetOption(FormattingOptions.TabSize, document.Project.Language);
var text = await document.GetTextAsync().ConfigureAwait(false);
var spans = IndentationHelper.GetSpansWithAlignedIndentation(text, classifiedSpanList.ToImmutableArray(), tabSize);
var textRuns = spans.Select(s => new ClassifiedTextRun(s.ClassificationType, snapshot.GetText(s.TextSpan.ToSpan()), ClassifiedTextRunStyle.UseClassificationFont));
if (textRuns.Any())
{
elements.Add(new ClassifiedTextElement(textRuns));
}
}
var content = new ContainerElement(
ContainerElementStyle.Stacked | ContainerElementStyle.VerticalPadding,
elements);
return(new IntellisenseQuickInfoItem(trackingSpan, content));
}
public void ReturnAnnotationClassifications() {
var code = @"
def f() -> int:
pass
";
using (var helper = new ClassifierHelper(code, PythonLanguageVersion.V35)) {
helper.CheckAnalysisClassifierSpans("f<f>c<int>");
}
}
