private void Tokenize(string document, SemanticTokensBuilder builder) { var faker = new Faker <TokenizationValue>() .RuleFor(z => z.type, f => f.PickRandom(SemanticTokenType.Defaults).OrNull(f, 0.2f) ?? new SemanticTokenType("none") ) .RuleFor(x => x.Modifiers, f => Enumerable.Range(0, f.Random.Int(0, 3)) .Select(z => f.PickRandom(SemanticTokenModifier.Defaults).OrNull(f, 0.2f) ?? new SemanticTokenModifier("none") ) .ToArray() .OrNull(f, 0.2f) ); foreach (var(line, text) in document.Split('\n').Select((text, line) => (line, text))) { var parts = text.TrimEnd().Split(';', ' ', '.', '"', '(', ')'); var index = 0; foreach (var part in parts) { faker.UseSeed(part.Length * line * text.Length); if (string.IsNullOrWhiteSpace(part)) { continue; } // _logger.LogWarning("Index for part before {Index}: {Text}", index, part); index = text.IndexOf(part, index, StringComparison.Ordinal); // _logger.LogInformation("Index for part after {Index}: {Text}", index, part); var item = faker.Generate(); builder.Push(line, index, part.Length, item.type, item.Modifiers); } } }
protected override async Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier, CancellationToken cancellationToken) { using var typesEnumerator = RotateEnum(SemanticTokenType.Defaults).GetEnumerator(); using var modifiersEnumerator = RotateEnum(SemanticTokenModifier.Defaults).GetEnumerator(); // you would normally get this from a common source that is managed by current open editor, current active editor, etc. var content = await File.ReadAllTextAsync(DocumentUri.GetFileSystemPath(identifier), cancellationToken); await Task.Yield(); foreach (var(line, text) in content.Split('\n').Select((text, line) => (line, text))) { var parts = text.TrimEnd().Split(';', ' ', '.', '"', '(', ')'); var index = 0; foreach (var part in parts) { typesEnumerator.MoveNext(); modifiersEnumerator.MoveNext(); if (string.IsNullOrWhiteSpace(part)) { continue; } index = text.IndexOf(part, index, StringComparison.Ordinal); builder.Push(line, index, part.Length, typesEnumerator.Current, modifiersEnumerator.Current); } } }
protected override async Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier, CancellationToken cancellationToken) { var ast = _bufferManager.GetAstFor(identifier.TextDocument.Uri.ToString()).Expressions.ToDotList(); var symbols = ExtractTypes <DotSymbol>(ast); // _logger.LogInformation( // $"-----------------extracted Symbols:\n{symbols.PrettyPrint()}"); // TODO: Make this parallel foreach (var symbol in symbols) { // _logger.LogInformation( // $"({symbol.Line}:{symbol.Column}) symbol: {symbol.Name}"); builder.Push(symbol.Line - 1, symbol.Column, symbol.Name.Length, SemanticTokenType.Function, SemanticTokenModifier.Static, SemanticTokenModifier.Documentation); } var strings = ExtractTypes <DotString>(ast); // _logger.LogInformation( // $"-----------------extracted Strings:\n{strings.PrettyPrint()}"); foreach (var str in strings) { // _logger.LogInformation( // $"({str.Line}:{str.Column}) string: {str.Value}"); builder.Push(str.Line - 1, str.Column, str.Value.Length + 2, SemanticTokenType.Class, SemanticTokenModifier.Static, SemanticTokenModifier.Readonly); } }
protected override Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier, CancellationToken cancellationToken) { ScriptFile file = _workspaceService.GetFile(identifier.TextDocument.Uri); foreach (Token token in file.ScriptTokens) { PushToken(token, builder); } return(Task.CompletedTask); }
protected override Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier, CancellationToken cancellationToken) { var compilationContext = this.compilationManager.GetCompilation(identifier.TextDocument.Uri); if (compilationContext != null) { SemanticTokenVisitor.BuildSemanticTokens(builder, compilationContext); } return(Task.CompletedTask); }
private static void PushToken(Token token, SemanticTokensBuilder builder) { foreach (SemanticToken sToken in ConvertToSemanticTokens(token)) { builder.Push( sToken.Line, sToken.Column, length: sToken.Text.Length, sToken.Type, tokenModifiers: sToken.TokenModifiers); } }
protected override async Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier, CancellationToken cancellationToken) { var doc = _lspHost.Documents[identifier.TextDocument.Uri]; if (doc.Compilation == null) { return; } foreach (var syntaxTree in doc.Compilation.SyntaxTrees) { TokenizeTokenTree(builder, syntaxTree.Root, cancellationToken); } }
protected override async Task Tokenize( SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier, CancellationToken cancellationToken ) { var hash = Hash.StringHash(identifier.TextDocument.Uri.GetFileSystemPath()); var now = DateTime.Now; IntPtr tokensPtr = IntPtr.Zero; int count = 0; long internalMicros = TreeSitter.GetTokens(hash, out tokensPtr, out count); // this can be async because it actually does work. var then = DateTime.Now; var elapsed = then - now; _logger.LogInformation("Elapsed time for C++ tokens: " + elapsed.TotalMilliseconds + " native time: " + internalMicros); List <Diagnostic> diagnostics = new List <Diagnostic>(); unsafe { SemanticToken *ptr = (SemanticToken *)tokensPtr; for (int i = 0; i < count; i++) { if ((int)ptr[i].type == 255) { Diagnostic diag = new Diagnostic(); diag.Severity = DiagnosticSeverity.Error; diag.Range = new OmniSharp.Extensions.LanguageServer.Protocol.Models.Range(); diag.Range.Start = new Position(ptr[i].line, ptr[i].col); diag.Range.End = new Position(ptr[i].line, ptr[i].col + ptr[i].length); diag.Message = "undeclared identifer"; diagnostics.Add(diag); continue; } builder.Push(ptr[i].line, ptr[i].col, ptr[i].length, (int)ptr[i].type, (int)ptr[i].modifier); } } diagnoser.Add(identifier.TextDocument.Uri, 0, diagnostics); diagnoser.Publish(identifier.TextDocument.Uri); }
private void TokenizeTokenTree(SemanticTokensBuilder builder, SyntaxNode token, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { return; } switch (token.Kind) { case SyntaxKind.SingleLineCommentTrivia: case SyntaxKind.MultiLineCommentTrivia: _logger.LogInformation($"Token: {token.Kind}. Line: {token.Location.StartLine}. Char: {token.Span.Start}. Len: {token.Span.Length}"); builder.Push(token.Location.StartLine, token.Location.StartCharacter, token.Span.Length, SemanticTokenType.Comment, SemanticTokenModifier.Defaults); break; case SyntaxKind.NumberToken: _logger.LogInformation($"Token: {token.Kind}. Line: {token.Location.StartLine}. Char: {token.Span.Start}. Len: {token.Span.Length}"); builder.Push(token.Location.StartLine, token.Location.StartCharacter, token.Span.Length, SemanticTokenType.Number, SemanticTokenModifier.Defaults); break; case SyntaxKind.StringToken: _logger.LogInformation($"Token: {token.Kind}. Line: {token.Location.StartLine}. Char: {token.Span.Start}. Len: {token.Span.Length}"); builder.Push(token.Location.StartLine, token.Location.StartCharacter, token.Span.Length, SemanticTokenType.String, SemanticTokenModifier.Defaults); break; case SyntaxKind.PlusToken: case SyntaxKind.PlusEqualsToken: case SyntaxKind.MinusToken: case SyntaxKind.MinusEqualsToken: case SyntaxKind.StarToken: case SyntaxKind.StarEqualsToken: case SyntaxKind.SlashToken: case SyntaxKind.SlashEqualsToken: case SyntaxKind.BangToken: case SyntaxKind.EqualsToken: case SyntaxKind.TildeToken: case SyntaxKind.HatToken: case SyntaxKind.HatEqualsToken: case SyntaxKind.AmpersandToken: case SyntaxKind.AmpersandAmpersandToken: case SyntaxKind.AmpersandEqualsToken: case SyntaxKind.PipeToken: case SyntaxKind.PipeEqualsToken: case SyntaxKind.PipePipeToken: case SyntaxKind.EqualsEqualsToken: case SyntaxKind.BangEqualsToken: case SyntaxKind.LessToken: case SyntaxKind.LessOrEqualsToken: case SyntaxKind.GreaterToken: case SyntaxKind.GreaterOrEqualsToken: _logger.LogInformation($"Token: {token.Kind}. Line: {token.Location.StartLine}. Char: {token.Span.Start}. Len: {token.Span.Length}"); builder.Push(token.Location.StartLine, token.Location.StartCharacter, token.Span.Length, SemanticTokenType.Operator, SemanticTokenModifier.Defaults); break; case SyntaxKind.IdentifierToken: _logger.LogInformation($"Token: {token.Kind}. Line: {token.Location.StartLine}. Char: {token.Span.Start}. Len: {token.Span.Length}"); // TODO: Be more specific builder.Push(token.Location.StartLine, token.Location.StartCharacter, token.Span.Length, SemanticTokenType.Member, SemanticTokenModifier.Defaults); break; case SyntaxKind.BreakKeyword: case SyntaxKind.ContinueKeyword: case SyntaxKind.ElseKeyword: case SyntaxKind.FalseKeyword: case SyntaxKind.ForKeyword: case SyntaxKind.FunctionKeyword: case SyntaxKind.IfKeyword: case SyntaxKind.LetKeyword: case SyntaxKind.ReturnKeyword: case SyntaxKind.ToKeyword: case SyntaxKind.TrueKeyword: case SyntaxKind.VarKeyword: case SyntaxKind.WhileKeyword: case SyntaxKind.DoKeyword: case SyntaxKind.StructKeyword: _logger.LogInformation($"Token: {token.Kind}. Line: {token.Location.StartLine}. Char: {token.Span.Start}. Len: {token.Span.Length}"); builder.Push(token.Location.StartLine, token.Location.StartCharacter, token.Span.Length, SemanticTokenType.Keyword, SemanticTokenModifier.Defaults); break; case SyntaxKind.Parameter: _logger.LogInformation($"Token: {token.Kind}. Line: {token.Location.StartLine}. Char: {token.Span.Start}. Len: {token.Span.Length}"); builder.Push(token.Location.StartLine, token.Location.StartCharacter, token.Span.Length, SemanticTokenType.Parameter, SemanticTokenModifier.Defaults); break; } // PERF: Don't do recursion foreach (var child in token.GetChildren()) { TokenizeTokenTree(builder, child, cancellationToken); } }
protected override Task Tokenize( SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier, CancellationToken cancellationToken ) => _tokenize(builder, identifier, _capability, cancellationToken);