protected override void PerformReclassification(IOilexerGrammarToken token, OilexerGrammarTokenType newClassification)
{
var currentTokenTag = this.tokens[token];
var newTokenTag = new GDTagSpan(currentTokenTag.Span, token, this.classificationTypes[newClassification]);
this.tokens[token] = newTokenTag;
}
public IEnumerable <GDTagSpan> TokensFrom(int index = 0, Span range = default(Span))
{
tokensInvalidatedCheck:
/* *
* If the tokens were invalidated due to a change in the text,
* clear the tokens and start again.
* */
if (this.TokensInvalidated)
{
/* *
* Indicate that the current stream
* of tokens is incomplete.
* */
this.FinishedLexing = false;
lock (this.tokens)
this.tokens.Clear();
this.TokensInvalidated = false;
}
int offset = 0;
/* *
* If the requested token index is greater
* than the number of cached tokens, and
* the last pass finished, yield a
* stopping point.
* */
if (index >= tokens.Count)
{
if (this.FinishedLexing)
{
yield break;
}
else
{
/* *
* Start scanning at the end of the
* cached elements.
* */
offset = this.tokens.Count;
goto nextToken;
}
}
else
{
GDTagSpan[] tokensArr;
/* *
* In the event that there's two machines going at once
* lock the current set and obtain a copy.
* *
* This ensures that the other machine doesn't interrupt
* the results of the current one because it changed the
*
* */
lock (this.tokens)
tokensArr = this.tokens.Values.ToArray();
int totalRetrieved = tokens.Count;
nextChunk:
foreach (var currentSpan in tokensArr)
{
if (this.TokensInvalidated)
{
goto tokensInvalidatedCheck;
}
else if (offset++ >= index)
{
if (range.End < currentSpan.Span.Start)
{
yield break;
}
if (CustomIntersect(currentSpan.Span, range))
{
yield return(currentSpan);
}
}
}
if (tokens.Count > totalRetrieved)
{
lock (this.tokens)
tokensArr = this.tokens.Values.Skip(totalRetrieved).ToArray();
totalRetrieved += tokensArr.Length;
goto nextChunk;
}
if (FinishedLexing)
{
yield break;
}
goto nextToken;
}
nextToken:
lock (TokensFromNextTokenLocker)
Lexer.NextToken();
/* *
* ToDo: Add code here to enable multi-threaded
* awareness, if two machines are shotgunning the
* next token method it might lead to two machines yielding
* different tokens sets
* */
/* *
* If the stream became dirty during parse, restart
* tokenization.
* */
if (this.TokensInvalidated)
{
goto tokensInvalidatedCheck;
}
var currentToken = Lexer.CurrentToken;
if (currentToken != null)
{
if (offset++ >= index)
{
var currentSpan = new Span((int)currentToken.Position, (int)currentToken.Length);
if (tokens.ContainsKey(currentToken))
{
yield break;
}
var currentTagSpan = new GDTagSpan(new SnapshotSpan(this.Buffer.CurrentSnapshot, currentSpan), currentToken, this.ClassificationTypes[currentToken.TokenType]);
lock (this.tokens)
{
if (!this.tokens.ContainsKey(currentToken))
{
this.tokens.Add(currentToken, currentTagSpan);
}
else
{
goto nextToken;
}
}
if (range.End < currentSpan.Start)
{
yield break;
}
if (CustomIntersect(currentSpan, range))
{
yield return(currentTagSpan);
}
}
goto nextToken;
}
this.FinishedLexing = true;
}
