public IEnumerable <ITagSpan <ClassificationTag> > GetTags(NormalizedSnapshotSpanCollection spans) { foreach (SnapshotSpan curSpan in spans) { int multilinePrefixSize; RustLexer lexer = InitLexer(curSpan, out multilinePrefixSize); IToken lastToken = null; for (int i = 0; true; i++) { int lastType = lastToken != null ? lastToken.Type : RustLexer.Eof; IToken token = lexer.NextToken(); if (token.Type == RustLexer.Eof) { if ((lastType == RustLexer.BLOCK_COMMENT || lastType == RustLexer.DOC_BLOCK_COMMENT) && !lastToken.Text.EndsWith("*/")) { UpdateMultilineDict(curSpan, lastType, RustLexer.BLOCK_COMMENT); } else if ((lastType == RustLexer.LIT_STR || lastType == RustLexer.LIT_STR_RAW || lastType == RustLexer.LIT_BINARY || lastType == RustLexer.LIT_BINARY_RAW) && !lastToken.Text.EndsWith("\"")) { UpdateMultilineDict(curSpan, lastType, RustLexer.LIT_STR); } else if (multilineTokens.Remove(curSpan.Snapshot.GetLineNumberFromPosition(curSpan.Start)) && curSpan.End.Position < curSpan.Snapshot.Length) { OnTagsChanged(new SnapshotSpan(curSpan.Snapshot, curSpan.End, 1)); } yield break; } lastToken = token; int realStart; int realTokenLength; if (i == 0) { realStart = curSpan.Start.Position + token.StartIndex; realTokenLength = token.Text.Length - multilinePrefixSize; } else { realStart = curSpan.Start.Position + token.StartIndex - multilinePrefixSize; realTokenLength = token.Text.Length; } // This little check is required because, we are sometimes called (eg. when copying in vs2015) on a span smaller than a whole line var tokenRange = new Span(realStart, realTokenLength).Intersection(curSpan); if (!tokenRange.HasValue) { continue; } var tokenSpan = new SnapshotSpan(curSpan.Snapshot, tokenRange.Value); yield return(new TagSpan <ClassificationTag>(tokenSpan, new ClassificationTag(_rustTypes[Utils.LexerTokenToRustToken(token.Text, token.Type)]))); } } }
public IEnumerable <ITagSpan <ClassificationTag> > GetTags(NormalizedSnapshotSpanCollection spans) { foreach (SnapshotSpan curSpan in spans) { int multilinePrefixSize; RustLexer lexer = InitLexer(curSpan, out multilinePrefixSize); IToken lastToken = null; for (int i = 0; true; i++) { int lastType = lastToken != null ? lastToken.Type : RustLexer.Eof; IToken token = lexer.NextToken(); if (token.Type == RustLexer.Eof) { if ((lastType == RustLexer.BLOCK_COMMENT || lastType == RustLexer.DOC_BLOCK_COMMENT) && !lastToken.Text.EndsWith("*/")) { UpdateMultilineDict(curSpan, lastType, RustLexer.BLOCK_COMMENT); } else if (lastType == RustLexer.LIT_STR || lastType == RustLexer.LIT_STR_RAW || lastType == RustLexer.LIT_BINARY || lastType == RustLexer.LIT_BINARY_RAW && !lastToken.Text.EndsWith("\"")) { UpdateMultilineDict(curSpan, lastType, RustLexer.LIT_STR); } else if (multilineTokens.Remove(curSpan.Snapshot.GetLineNumberFromPosition(curSpan.Start)) && curSpan.End.Position < curSpan.Snapshot.Length) { OnTagsChanged(new SnapshotSpan(curSpan.Snapshot, curSpan.End, 1)); } yield break; } lastToken = token; int realStart; int realTokenLength; if (i == 0) { realStart = curSpan.Start.Position + token.StartIndex; realTokenLength = token.Text.Length - multilinePrefixSize; } else { realStart = curSpan.Start.Position + token.StartIndex - multilinePrefixSize; realTokenLength = token.Text.Length; } var tokenSpan = new SnapshotSpan(curSpan.Snapshot, new Span(realStart, realTokenLength)); yield return(new TagSpan <ClassificationTag>(tokenSpan, new ClassificationTag(_rustTypes[Utils.LexerTokenToRustToken(token.Text, token.Type)]))); } } }
public static IEnumerable <Antlr4.Runtime.IToken> LexString(string text) { var lexer = new RustLexer(text); while (true) { var tok = lexer.NextToken(); if (tok.Type == RustLexer.Eof) { yield break; } else { yield return(tok); } } }
public IEnumerable <ITagSpan <RustTokenTag> > GetTags(NormalizedSnapshotSpanCollection spans) { foreach (SnapshotSpan curSpan in spans) { var containingLine = curSpan.Start.GetContainingLine(); var lexer = new RustLexer(curSpan.Start.GetContainingLine().GetText()); int curLoc = containingLine.Start.Position; while (true) { var tok = lexer.NextToken(); if (tok.Type == RustLexer.Eof) { yield break; } var tokenSpan = new SnapshotSpan(curSpan.Snapshot, new Span(curLoc + tok.StartIndex, tok.Text.Length)); if (tokenSpan.IntersectsWith(curSpan)) { yield return(new TagSpan <RustTokenTag>(tokenSpan, new RustTokenTag(Utils.LexerTokenToRustToken(tok.Text, tok.Type)))); } } } }