private void PerformReTag() { _lastTags.Clear(); SLang.Reader reader = new SLang.Reader(_snapshot.GetText()); SLang.Tokenizer tokenizer = new SLang.Tokenizer(reader, (SLang.Options)null); SLang.Token token = tokenizer.getNextToken(); while (token.code != SLang.TokenCode.EOS) { SLangTokenTag currentTag = new SLangTokenTag(token); if (currentTag.type != SLangTokenType.Whitespace) { Span currentTokenSpan = ConvertToSpan(token.span, _snapshot); SnapshotSpan tokenSpan = new SnapshotSpan(_snapshot, currentTokenSpan); _lastTags.Add(new TagSpan <SLangTokenTag>(tokenSpan, currentTag)); } token = tokenizer.getNextToken(); } }
public SLangTokenTag(SLang.Token token) { this.token = token; this.type = Classification.ClassificationMapping.getTokenType(token.code); }