protected override void PerformReclassification(IOilexerGrammarToken token, OilexerGrammarTokenType newClassification) { var currentTokenTag = this.tokens[token]; var newTokenTag = new GDTagSpan(currentTokenTag.Span, token, this.classificationTypes[newClassification]); this.tokens[token] = newTokenTag; }
public IEnumerable <GDTagSpan> TokensFrom(int index = 0, Span range = default(Span)) { tokensInvalidatedCheck: /* * * If the tokens were invalidated due to a change in the text, * clear the tokens and start again. * */ if (this.TokensInvalidated) { /* * * Indicate that the current stream * of tokens is incomplete. * */ this.FinishedLexing = false; lock (this.tokens) this.tokens.Clear(); this.TokensInvalidated = false; } int offset = 0; /* * * If the requested token index is greater * than the number of cached tokens, and * the last pass finished, yield a * stopping point. * */ if (index >= tokens.Count) { if (this.FinishedLexing) { yield break; } else { /* * * Start scanning at the end of the * cached elements. * */ offset = this.tokens.Count; goto nextToken; } } else { GDTagSpan[] tokensArr; /* * * In the event that there's two machines going at once * lock the current set and obtain a copy. * * * This ensures that the other machine doesn't interrupt * the results of the current one because it changed the * * */ lock (this.tokens) tokensArr = this.tokens.Values.ToArray(); int totalRetrieved = tokens.Count; nextChunk: foreach (var currentSpan in tokensArr) { if (this.TokensInvalidated) { goto tokensInvalidatedCheck; } else if (offset++ >= index) { if (range.End < currentSpan.Span.Start) { yield break; } if (CustomIntersect(currentSpan.Span, range)) { yield return(currentSpan); } } } if (tokens.Count > totalRetrieved) { lock (this.tokens) tokensArr = this.tokens.Values.Skip(totalRetrieved).ToArray(); totalRetrieved += tokensArr.Length; goto nextChunk; } if (FinishedLexing) { yield break; } goto nextToken; } nextToken: lock (TokensFromNextTokenLocker) Lexer.NextToken(); /* * * ToDo: Add code here to enable multi-threaded * awareness, if two machines are shotgunning the * next token method it might lead to two machines yielding * different tokens sets * */ /* * * If the stream became dirty during parse, restart * tokenization. * */ if (this.TokensInvalidated) { goto tokensInvalidatedCheck; } var currentToken = Lexer.CurrentToken; if (currentToken != null) { if (offset++ >= index) { var currentSpan = new Span((int)currentToken.Position, (int)currentToken.Length); if (tokens.ContainsKey(currentToken)) { yield break; } var currentTagSpan = new GDTagSpan(new SnapshotSpan(this.Buffer.CurrentSnapshot, currentSpan), currentToken, this.ClassificationTypes[currentToken.TokenType]); lock (this.tokens) { if (!this.tokens.ContainsKey(currentToken)) { this.tokens.Add(currentToken, currentTagSpan); } else { goto nextToken; } } if (range.End < currentSpan.Start) { yield break; } if (CustomIntersect(currentSpan, range)) { yield return(currentTagSpan); } } goto nextToken; } this.FinishedLexing = true; }