protected virtual IToken PeekToken(int offset, bool skipOffChannelTokens, out TState stateAfterToken)
{
stateAfterToken = default(TState);
int index = _tokenIndex + offset;
while (index >= 0 && (_tokenCache.Count == 0 || _tokenCache[_tokenCache.Count - 1].Item1.Type != IntStreamConstants.Eof))
{
while (index >= _tokenCache.Count)
{
if (_tokenCache.Count != 0 && _tokenCache[_tokenCache.Count - 1].Item1.Type == IntStreamConstants.Eof)
{
break;
}
IToken token = _lexer.NextToken();
TState state = _lexer.GetCurrentState();
_tokenCache.Add(Tuple.Create(token, state));
}
if (index >= _tokenCache.Count)
{
index = _tokenCache.Count - 1;
}
IToken t = _tokenCache[index].Item1;
if (skipOffChannelTokens && t.Channel != Lexer.DefaultTokenChannel)
{
if (offset < 0)
{
index--;
}
else
{
index++;
}
continue;
}
stateAfterToken = _tokenCache[index].Item2;
return(t);
}
return(null);
}
public virtual IList <ClassificationSpan> GetClassificationSpans(SnapshotSpan span)
{
List <ClassificationSpan> classificationSpans = new List <ClassificationSpan>();
if (_failedTimeout)
{
return(classificationSpans);
}
bool spanExtended = false;
int extendMultilineSpanToLine = 0;
SnapshotSpan extendedSpan = span;
ITextSnapshot snapshot = span.Snapshot;
ClassifierState classifierState = _lineStatesCache.GetValue(snapshot, CreateClassifierState);
using (_lock.UpgradableReadLock(TimeSpan.FromMilliseconds(250)))
{
Span requestedSpan = span;
TState startState = AdjustParseSpan(classifierState, ref span);
ICharStream input = CreateInputStream(span);
ITokenSourceWithState <TState> lexer = CreateLexer(input, span.Start.GetContainingLine().LineNumber + 1, startState);
lexer.TokenFactory = new SnapshotTokenFactory(snapshot, GetEffectiveTokenSource(lexer));
IToken previousToken = null;
bool previousTokenEndsLine = false;
/* this is held outside the loop because only tokens which end at the end of a line
* impact its value.
*/
bool lineStateChanged = false;
while (true)
{
IToken token = lexer.NextToken();
// The latter is true for EOF token with span.End at the end of the document
bool inBounds = token.StartIndex < span.End.Position ||
token.StopIndex < span.End.Position;
int startLineCurrent;
if (token.Type == IntStreamConstants.Eof)
{
startLineCurrent = span.Snapshot.LineCount - 1;
}
else
{
startLineCurrent = token.Line - 1;
}
// endLinePrevious is the line number the previous token ended on
int endLinePrevious;
if (previousToken != null)
{
Debug.Assert(previousToken.StopIndex >= previousToken.StartIndex, "previousToken can't be EOF");
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex);
}
else
{
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1;
}
if (startLineCurrent > endLinePrevious + 1 || (startLineCurrent == endLinePrevious + 1 && !previousTokenEndsLine))
{
int firstMultilineLine = endLinePrevious;
if (previousToken == null || previousTokenEndsLine)
{
firstMultilineLine++;
}
for (int i = firstMultilineLine; i < startLineCurrent; i++)
{
if (!classifierState._lineStates[i].MultilineToken || lineStateChanged)
{
extendMultilineSpanToLine = i + 1;
}
SetLineState(classifierState, i, LineStateInfo.Multiline);
}
}
if (IsMultilineToken(span.Snapshot, lexer, token))
{
int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex);
int stopLine = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex));
for (int i = startLine; i < stopLine; i++)
{
if (!classifierState._lineStates[i].MultilineToken)
{
extendMultilineSpanToLine = i + 1;
}
SetLineState(classifierState, i, LineStateInfo.Multiline);
}
}
bool tokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token);
if (tokenEndsLine)
{
TState stateAtEndOfLine = lexer.GetCurrentState();
int line = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex));
lineStateChanged =
classifierState._lineStates[line].MultilineToken ||
!_stateComparer.Equals(classifierState._lineStates[line].EndLineState, stateAtEndOfLine);
// even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated.
SetLineState(classifierState, line, new LineStateInfo(stateAtEndOfLine));
if (lineStateChanged)
{
if (line < span.Snapshot.LineCount - 1)
{
/* update the span's end position or the line state change won't be reflected
* in the editor
*/
int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
}
}
if (token.Type == IntStreamConstants.Eof)
{
break;
}
if (token.StartIndex >= span.End.Position)
{
break;
}
previousToken = token;
previousTokenEndsLine = tokenEndsLine;
if (token.StopIndex < requestedSpan.Start)
{
continue;
}
var tokenClassificationSpans = GetClassificationSpansForToken(token, span.Snapshot);
if (tokenClassificationSpans != null)
{
classificationSpans.AddRange(tokenClassificationSpans);
}
if (!inBounds)
{
break;
}
}
}
if (extendMultilineSpanToLine > 0)
{
int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
if (spanExtended)
{
/* Subtract 1 from each of these because the spans include the line break on their last
* line, forcing it to appear as the first position on the following line.
*/
int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End);
int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1;
// when considering the last line of a document, span and extendedSpan may end on the same line
ForceReclassifyLines(classifierState, firstLine, Math.Max(firstLine, lastLine));
}
return(classificationSpans);
}
public virtual IList <ClassificationSpan> GetClassificationSpans(SnapshotSpan span)
{
Contract.Ensures(Contract.Result <IList <ClassificationSpan> >() != null);
Span requestedSpan = span;
TState startState = AdjustParseSpan(ref span);
ICharStream input = CreateInputStream(span);
ITokenSourceWithState <TState> lexer = CreateLexer(input, startState);
List <ClassificationSpan> classificationSpans = new List <ClassificationSpan>();
IToken previousToken = null;
bool previousTokenEndsLine = false;
/* this is held outside the loop because only tokens which end at the end of a line
* impact its value.
*/
bool lineStateChanged = false;
int extendMultilineSpanToLine = 0;
SnapshotSpan extendedSpan = span;
bool spanExtended = false;
while (true)
{
IToken token = lexer.NextToken();
bool inBounds = token.StartIndex < span.End.Position;
int startLineCurrent;
if (token.Type == CharStreamConstants.EndOfFile)
{
startLineCurrent = span.Snapshot.LineCount;
}
else
{
startLineCurrent = token.Line;
}
if (previousToken == null || previousToken.Line < startLineCurrent - 1)
{
// endLinePrevious is the line number the previous token ended on
int endLinePrevious;
if (previousToken != null)
{
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex + 1);
}
else
{
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1;
}
if (startLineCurrent > endLinePrevious + 1)
{
int firstMultilineLine = endLinePrevious;
if (previousToken == null || previousTokenEndsLine)
{
firstMultilineLine++;
}
for (int i = firstMultilineLine; i < startLineCurrent; i++)
{
if (!_lineStates[i].MultilineToken || lineStateChanged)
{
extendMultilineSpanToLine = i + 1;
}
if (inBounds)
{
SetLineState(i, LineStateInfo.Multiline);
}
}
}
}
if (token.Type == CharStreamConstants.EndOfFile)
{
break;
}
previousToken = token;
previousTokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token);
if (IsMultilineToken(span.Snapshot, lexer, token))
{
int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex);
int stopLine = span.Snapshot.GetLineNumberFromPosition(token.StopIndex + 1);
for (int i = startLine; i < stopLine; i++)
{
if (!_lineStates[i].MultilineToken)
{
extendMultilineSpanToLine = i + 1;
}
if (inBounds)
{
SetLineState(i, LineStateInfo.Multiline);
}
}
}
bool tokenEndsLine = previousTokenEndsLine;
if (tokenEndsLine)
{
TState stateAtEndOfLine = lexer.GetCurrentState();
int line = span.Snapshot.GetLineNumberFromPosition(token.StopIndex + 1);
lineStateChanged =
_lineStates[line].MultilineToken ||
!_stateComparer.Equals(_lineStates[line].EndLineState, stateAtEndOfLine);
// even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated.
if (inBounds)
{
SetLineState(line, new LineStateInfo(stateAtEndOfLine));
}
if (lineStateChanged)
{
if (line < span.Snapshot.LineCount - 1)
{
/* update the span's end position or the line state change won't be reflected
* in the editor
*/
int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
}
}
if (token.StartIndex >= span.End.Position)
{
break;
}
if (token.StopIndex < requestedSpan.Start)
{
continue;
}
var tokenClassificationSpans = GetClassificationSpansForToken(token, span.Snapshot);
if (tokenClassificationSpans != null)
{
classificationSpans.AddRange(tokenClassificationSpans);
}
if (!inBounds)
{
break;
}
}
if (extendMultilineSpanToLine > 0)
{
int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
if (spanExtended)
{
/* Subtract 1 from each of these because the spans include the line break on their last
* line, forcing it to appear as the first position on the following line.
*/
int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End);
int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1;
ForceReclassifyLines(firstLine, lastLine);
}
return(classificationSpans);
}