protected virtual bool TokenEndsAtEndOfLine(ITextSnapshot snapshot, ITokenSourceWithState <TState> lexer, IToken token)
{
if (token.StopIndex + 1 >= snapshot.Length)
{
return(true);
}
char c = snapshot[token.StopIndex + 1];
return(c == '\r' || c == '\n');
}
protected virtual bool IsMultilineToken(ITextSnapshot snapshot, ITokenSourceWithState <TState> lexer, IToken token)
{
if (token.Type == IntStreamConstants.Eof)
{
return(false);
}
int startLine = snapshot.GetLineNumberFromPosition(token.StartIndex);
int stopLine = snapshot.GetLineNumberFromPosition(token.StopIndex);
return(startLine != stopLine);
}
protected virtual bool TokenEndsAtEndOfLine(ITextSnapshot snapshot, ITokenSourceWithState <TState> lexer, IToken token)
{
ICharStream charStream = lexer.CharStream;
if (charStream != null)
{
int nextCharIndex = token.StopIndex + 1;
if (nextCharIndex >= charStream.Size)
{
return(true);
}
int c = charStream.GetText(new Interval(token.StopIndex + 1, token.StopIndex + 1))[0];
return(c == '\r' || c == '\n');
}
ITextSnapshotLine line = snapshot.GetLineFromPosition(token.StopIndex + 1);
return(line.End <= token.StopIndex + 1 && line.EndIncludingLineBreak >= token.StopIndex + 1);
}
protected virtual IEnumerable <ITagSpan <TTag> > GetTags(SnapshotSpan span)
{
List <ITagSpan <TTag> > classificationSpans = new List <ITagSpan <TTag> >();
bool spanExtended = false;
int extendMultilineSpanToLine = 0;
SnapshotSpan extendedSpan = span;
Span requestedSpan = span;
TState startState = AdjustParseSpan(ref span);
ITokenSourceWithState <TState> lexer = CreateLexer(span, span.Start.GetContainingLine().LineNumber + 1, startState);
IToken previousToken = null;
bool previousTokenEndsLine = false;
/* this is held outside the loop because only tokens which end at the end of a line
* impact its value.
*/
bool lineStateChanged = false;
_lexer = lexer;
_tokenCache = new List <Tuple <IToken, TState> >();
_tokenIndex = -1;
while (true)
{
_tokenIndex++;
TState stateAfterToken;
IToken token = PeekToken(0, false, out stateAfterToken);
bool inBounds = token.StartIndex < span.End.Position;
int startLineCurrent;
if (token.Type == IntStreamConstants.Eof)
{
startLineCurrent = span.Snapshot.LineCount;
}
else
{
startLineCurrent = token.Line;
}
if (previousToken == null || previousToken.Line < startLineCurrent - 1)
{
// endLinePrevious is the line number the previous token ended on
int endLinePrevious;
if (previousToken != null)
{
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex + 1);
}
else
{
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1;
}
if (startLineCurrent > endLinePrevious + 1)
{
int firstMultilineLine = endLinePrevious;
if (previousToken == null || previousTokenEndsLine)
{
firstMultilineLine++;
}
for (int i = firstMultilineLine; i < startLineCurrent; i++)
{
if (!_lineStates[i].IsMultiline || lineStateChanged)
{
extendMultilineSpanToLine = i + 1;
}
if (inBounds)
{
SetLineState(i, GetStartState().CreateMultilineState());
}
}
}
}
if (token.Type == IntStreamConstants.Eof)
{
break;
}
previousToken = token;
previousTokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token);
if (IsMultilineToken(span.Snapshot, lexer, token))
{
int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex);
int stopLine = span.Snapshot.GetLineNumberFromPosition(token.StopIndex + 1);
for (int i = startLine; i < stopLine; i++)
{
if (!_lineStates[i].IsMultiline)
{
extendMultilineSpanToLine = i + 1;
}
if (inBounds)
{
SetLineState(i, GetStartState().CreateMultilineState());
}
}
}
bool tokenEndsLine = previousTokenEndsLine;
if (tokenEndsLine)
{
TState stateAtEndOfLine = stateAfterToken;
int line = span.Snapshot.GetLineNumberFromPosition(token.StopIndex + 1);
lineStateChanged =
_lineStates[line].IsMultiline ||
!_stateComparer.Equals(_lineStates[line], stateAtEndOfLine);
// even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated.
if (inBounds)
{
SetLineState(line, stateAtEndOfLine);
}
if (lineStateChanged)
{
if (line < span.Snapshot.LineCount - 1)
{
/* update the span's end position or the line state change won't be reflected
* in the editor
*/
int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
}
}
if (token.StartIndex >= span.End.Position)
{
break;
}
if (token.StopIndex < requestedSpan.Start)
{
continue;
}
var tokenClassificationSpans = GetTagSpansForToken(token, span.Snapshot);
if (tokenClassificationSpans != null)
{
classificationSpans.AddRange(tokenClassificationSpans);
}
if (!inBounds)
{
break;
}
}
if (extendMultilineSpanToLine > 0)
{
int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
if (spanExtended)
{
/* Subtract 1 from each of these because the spans include the line break on their last
* line, forcing it to appear as the first position on the following line.
*/
int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End);
int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1;
ForceRetagLines(firstLine, lastLine);
}
return(classificationSpans);
}
protected virtual bool IsMultilineToken(ITextSnapshot snapshot, ITokenSourceWithState <TState> lexer, IToken token)
{
ITextSnapshotLine line = snapshot.GetLineFromPosition(token.StartIndex);
return(token.StopIndex > line.End);
}
public virtual IList <ClassificationSpan> GetClassificationSpans(SnapshotSpan span)
{
List <ClassificationSpan> classificationSpans = new List <ClassificationSpan>();
if (_failedTimeout)
{
return(classificationSpans);
}
bool spanExtended = false;
int extendMultilineSpanToLine = 0;
SnapshotSpan extendedSpan = span;
ITextSnapshot snapshot = span.Snapshot;
ClassifierState classifierState = _lineStatesCache.GetValue(snapshot, CreateClassifierState);
using (_lock.UpgradableReadLock(TimeSpan.FromMilliseconds(250)))
{
Span requestedSpan = span;
TState startState = AdjustParseSpan(classifierState, ref span);
ICharStream input = CreateInputStream(span);
ITokenSourceWithState <TState> lexer = CreateLexer(input, span.Start.GetContainingLine().LineNumber + 1, startState);
lexer.TokenFactory = new SnapshotTokenFactory(snapshot, GetEffectiveTokenSource(lexer));
IToken previousToken = null;
bool previousTokenEndsLine = false;
/* this is held outside the loop because only tokens which end at the end of a line
* impact its value.
*/
bool lineStateChanged = false;
while (true)
{
IToken token = lexer.NextToken();
// The latter is true for EOF token with span.End at the end of the document
bool inBounds = token.StartIndex < span.End.Position ||
token.StopIndex < span.End.Position;
int startLineCurrent;
if (token.Type == IntStreamConstants.Eof)
{
startLineCurrent = span.Snapshot.LineCount - 1;
}
else
{
startLineCurrent = token.Line - 1;
}
// endLinePrevious is the line number the previous token ended on
int endLinePrevious;
if (previousToken != null)
{
Debug.Assert(previousToken.StopIndex >= previousToken.StartIndex, "previousToken can't be EOF");
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex);
}
else
{
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1;
}
if (startLineCurrent > endLinePrevious + 1 || (startLineCurrent == endLinePrevious + 1 && !previousTokenEndsLine))
{
int firstMultilineLine = endLinePrevious;
if (previousToken == null || previousTokenEndsLine)
{
firstMultilineLine++;
}
for (int i = firstMultilineLine; i < startLineCurrent; i++)
{
if (!classifierState._lineStates[i].MultilineToken || lineStateChanged)
{
extendMultilineSpanToLine = i + 1;
}
SetLineState(classifierState, i, LineStateInfo.Multiline);
}
}
if (IsMultilineToken(span.Snapshot, lexer, token))
{
int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex);
int stopLine = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex));
for (int i = startLine; i < stopLine; i++)
{
if (!classifierState._lineStates[i].MultilineToken)
{
extendMultilineSpanToLine = i + 1;
}
SetLineState(classifierState, i, LineStateInfo.Multiline);
}
}
bool tokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token);
if (tokenEndsLine)
{
TState stateAtEndOfLine = lexer.GetCurrentState();
int line = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex));
lineStateChanged =
classifierState._lineStates[line].MultilineToken ||
!_stateComparer.Equals(classifierState._lineStates[line].EndLineState, stateAtEndOfLine);
// even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated.
SetLineState(classifierState, line, new LineStateInfo(stateAtEndOfLine));
if (lineStateChanged)
{
if (line < span.Snapshot.LineCount - 1)
{
/* update the span's end position or the line state change won't be reflected
* in the editor
*/
int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
}
}
if (token.Type == IntStreamConstants.Eof)
{
break;
}
if (token.StartIndex >= span.End.Position)
{
break;
}
previousToken = token;
previousTokenEndsLine = tokenEndsLine;
if (token.StopIndex < requestedSpan.Start)
{
continue;
}
var tokenClassificationSpans = GetClassificationSpansForToken(token, span.Snapshot);
if (tokenClassificationSpans != null)
{
classificationSpans.AddRange(tokenClassificationSpans);
}
if (!inBounds)
{
break;
}
}
}
if (extendMultilineSpanToLine > 0)
{
int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
if (spanExtended)
{
/* Subtract 1 from each of these because the spans include the line break on their last
* line, forcing it to appear as the first position on the following line.
*/
int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End);
int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1;
// when considering the last line of a document, span and extendedSpan may end on the same line
ForceReclassifyLines(classifierState, firstLine, Math.Max(firstLine, lastLine));
}
return(classificationSpans);
}
protected virtual ITokenSource GetEffectiveTokenSource(ITokenSourceWithState <TState> lexer)
{
return(lexer);
}
protected virtual ITokenSource GetEffectiveTokenSource(ITokenSourceWithState <TState> lexer)
{
Contract.Ensures(Contract.Result <ITokenSource>() != null);
return(lexer);
}
