protected override async Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier,
CancellationToken cancellationToken)
{
using var typesEnumerator = RotateEnum(SemanticTokenType.Defaults).GetEnumerator();
using var modifiersEnumerator = RotateEnum(SemanticTokenModifier.Defaults).GetEnumerator();
// you would normally get this from a common source that is managed by current open editor, current active editor, etc.
var content = await File.ReadAllTextAsync(DocumentUri.GetFileSystemPath(identifier), cancellationToken);
await Task.Yield();
foreach (var(line, text) in content.Split('\n').Select((text, line) => (line, text)))
{
var parts = text.TrimEnd().Split(';', ' ', '.', '"', '(', ')');
var index = 0;
foreach (var part in parts)
{
typesEnumerator.MoveNext();
modifiersEnumerator.MoveNext();
if (string.IsNullOrWhiteSpace(part))
{
continue;
}
index = text.IndexOf(part, index, StringComparison.Ordinal);
builder.Push(line, index, part.Length, typesEnumerator.Current, modifiersEnumerator.Current);
}
}
}
protected override async Task Tokenize(SemanticTokensBuilder builder,
ITextDocumentIdentifierParams identifier,
CancellationToken cancellationToken)
{
var ast =
_bufferManager.GetAstFor(identifier.TextDocument.Uri.ToString()).Expressions.ToDotList();
var symbols = ExtractTypes <DotSymbol>(ast);
// _logger.LogInformation(
// $"-----------------extracted Symbols:\n{symbols.PrettyPrint()}");
// TODO: Make this parallel
foreach (var symbol in symbols)
{
// _logger.LogInformation(
// $"({symbol.Line}:{symbol.Column}) symbol: {symbol.Name}");
builder.Push(symbol.Line - 1, symbol.Column,
symbol.Name.Length,
SemanticTokenType.Function, SemanticTokenModifier.Static,
SemanticTokenModifier.Documentation);
}
var strings = ExtractTypes <DotString>(ast);
// _logger.LogInformation(
// $"-----------------extracted Strings:\n{strings.PrettyPrint()}");
foreach (var str in strings)
{
// _logger.LogInformation(
// $"({str.Line}:{str.Column}) string: {str.Value}");
builder.Push(str.Line - 1, str.Column, str.Value.Length + 2,
SemanticTokenType.Class, SemanticTokenModifier.Static,
SemanticTokenModifier.Readonly);
}
}
protected override Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier,
CancellationToken cancellationToken)
{
ScriptFile file = _workspaceService.GetFile(identifier.TextDocument.Uri);
foreach (Token token in file.ScriptTokens)
{
PushToken(token, builder);
}
return(Task.CompletedTask);
}
protected override Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier, CancellationToken cancellationToken)
{
var compilationContext = this.compilationManager.GetCompilation(identifier.TextDocument.Uri);
if (compilationContext != null)
{
SemanticTokenVisitor.BuildSemanticTokens(builder, compilationContext);
}
return(Task.CompletedTask);
}
protected override async Task Tokenize(SemanticTokensBuilder builder,
ITextDocumentIdentifierParams identifier,
CancellationToken cancellationToken)
{
var doc = _lspHost.Documents[identifier.TextDocument.Uri];
if (doc.Compilation == null)
{
return;
}
foreach (var syntaxTree in doc.Compilation.SyntaxTrees)
{
TokenizeTokenTree(builder, syntaxTree.Root, cancellationToken);
}
}
protected override async Task Tokenize(
SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier,
CancellationToken cancellationToken
)
{
var hash = Hash.StringHash(identifier.TextDocument.Uri.GetFileSystemPath());
var now = DateTime.Now;
IntPtr tokensPtr = IntPtr.Zero;
int count = 0;
long internalMicros = TreeSitter.GetTokens(hash, out tokensPtr, out count); // this can be async because it actually does work.
var then = DateTime.Now;
var elapsed = then - now;
_logger.LogInformation("Elapsed time for C++ tokens: " + elapsed.TotalMilliseconds + " native time: " + internalMicros);
List <Diagnostic> diagnostics = new List <Diagnostic>();
unsafe
{
SemanticToken *ptr = (SemanticToken *)tokensPtr;
for (int i = 0; i < count; i++)
{
if ((int)ptr[i].type == 255)
{
Diagnostic diag = new Diagnostic();
diag.Severity = DiagnosticSeverity.Error;
diag.Range = new OmniSharp.Extensions.LanguageServer.Protocol.Models.Range();
diag.Range.Start = new Position(ptr[i].line, ptr[i].col);
diag.Range.End = new Position(ptr[i].line, ptr[i].col + ptr[i].length);
diag.Message = "undeclared identifer";
diagnostics.Add(diag);
continue;
}
builder.Push(ptr[i].line, ptr[i].col, ptr[i].length, (int)ptr[i].type, (int)ptr[i].modifier);
}
}
diagnoser.Add(identifier.TextDocument.Uri, 0, diagnostics);
diagnoser.Publish(identifier.TextDocument.Uri);
}
protected override Task <SemanticTokensDocument> GetSemanticTokensDocument(
ITextDocumentIdentifierParams @params,
CancellationToken cancellationToken)
{
return(Task.FromResult(new SemanticTokensDocument(RegistrationOptions.Legend)));
}
protected abstract Task <SemanticTokensDocument> GetSemanticTokensDocument(
ITextDocumentIdentifierParams @params, CancellationToken cancellationToken);
protected abstract Task Tokenize(
SemanticTokensBuilder builder,
ITextDocumentIdentifierParams identifier,
CancellationToken cancellationToken);
protected override Task <SemanticTokensDocument> GetSemanticTokensDocument(
ITextDocumentIdentifierParams @params, CancellationToken cancellationToken) =>
_getSemanticTokensDocument(@params, cancellationToken);
protected override Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier,
CancellationToken cancellationToken) =>
_tokenize(builder, identifier, cancellationToken);
GetSemanticTokensDocument(ITextDocumentIdentifierParams @params, CancellationToken cancellationToken)
{
return(Task.FromResult(new SemanticTokensDocument(GetRegistrationOptions().Legend)));
}
/// <summary> /// Get the local file-system path for the specified document URI. /// </summary> /// <param name="textDocumentIdentifierParams"> /// The text document params object /// </param> /// <returns> /// The file-system path, or <c>null</c> if the URI does not represent a file-system path. /// </returns> public static string GetFileSystemPath(ITextDocumentIdentifierParams textDocumentIdentifierParams) => GetFileSystemPath(textDocumentIdentifierParams.TextDocument.Uri);
/// <summary>
/// Get the local file-system path for the specified document URI.
/// </summary>
/// <param name="textDocumentIdentifierParams">
/// The text document params object
/// </param>
/// <returns>
/// The file-system path, or <c>null</c> if the URI does not represent a file-system path.
/// </returns>
public static string?GetFileSystemPath(ITextDocumentIdentifierParams textDocumentIdentifierParams)
{
return(GetFileSystemPath(textDocumentIdentifierParams.TextDocument.Uri));
}
