protected virtual IToken PeekToken(int offset, bool skipOffChannelTokens, out TState stateAfterToken) { stateAfterToken = default(TState); int index = _tokenIndex + offset; while (index >= 0 && (_tokenCache.Count == 0 || _tokenCache[_tokenCache.Count - 1].Item1.Type != IntStreamConstants.Eof)) { while (index >= _tokenCache.Count) { if (_tokenCache.Count != 0 && _tokenCache[_tokenCache.Count - 1].Item1.Type == IntStreamConstants.Eof) { break; } IToken token = _lexer.NextToken(); TState state = _lexer.GetCurrentState(); _tokenCache.Add(Tuple.Create(token, state)); } if (index >= _tokenCache.Count) { index = _tokenCache.Count - 1; } IToken t = _tokenCache[index].Item1; if (skipOffChannelTokens && t.Channel != Lexer.DefaultTokenChannel) { if (offset < 0) { index--; } else { index++; } continue; } stateAfterToken = _tokenCache[index].Item2; return(t); } return(null); }
public virtual IList <ClassificationSpan> GetClassificationSpans(SnapshotSpan span) { List <ClassificationSpan> classificationSpans = new List <ClassificationSpan>(); if (_failedTimeout) { return(classificationSpans); } bool spanExtended = false; int extendMultilineSpanToLine = 0; SnapshotSpan extendedSpan = span; ITextSnapshot snapshot = span.Snapshot; ClassifierState classifierState = _lineStatesCache.GetValue(snapshot, CreateClassifierState); using (_lock.UpgradableReadLock(TimeSpan.FromMilliseconds(250))) { Span requestedSpan = span; TState startState = AdjustParseSpan(classifierState, ref span); ICharStream input = CreateInputStream(span); ITokenSourceWithState <TState> lexer = CreateLexer(input, span.Start.GetContainingLine().LineNumber + 1, startState); lexer.TokenFactory = new SnapshotTokenFactory(snapshot, GetEffectiveTokenSource(lexer)); IToken previousToken = null; bool previousTokenEndsLine = false; /* this is held outside the loop because only tokens which end at the end of a line * impact its value. */ bool lineStateChanged = false; while (true) { IToken token = lexer.NextToken(); // The latter is true for EOF token with span.End at the end of the document bool inBounds = token.StartIndex < span.End.Position || token.StopIndex < span.End.Position; int startLineCurrent; if (token.Type == IntStreamConstants.Eof) { startLineCurrent = span.Snapshot.LineCount - 1; } else { startLineCurrent = token.Line - 1; } // endLinePrevious is the line number the previous token ended on int endLinePrevious; if (previousToken != null) { Debug.Assert(previousToken.StopIndex >= previousToken.StartIndex, "previousToken can't be EOF"); endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex); } else { endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1; } if (startLineCurrent > endLinePrevious + 1 || (startLineCurrent == endLinePrevious + 1 && !previousTokenEndsLine)) { int firstMultilineLine = endLinePrevious; if (previousToken == null || previousTokenEndsLine) { firstMultilineLine++; } for (int i = firstMultilineLine; i < startLineCurrent; i++) { if (!classifierState._lineStates[i].MultilineToken || lineStateChanged) { extendMultilineSpanToLine = i + 1; } SetLineState(classifierState, i, LineStateInfo.Multiline); } } if (IsMultilineToken(span.Snapshot, lexer, token)) { int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex); int stopLine = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex)); for (int i = startLine; i < stopLine; i++) { if (!classifierState._lineStates[i].MultilineToken) { extendMultilineSpanToLine = i + 1; } SetLineState(classifierState, i, LineStateInfo.Multiline); } } bool tokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token); if (tokenEndsLine) { TState stateAtEndOfLine = lexer.GetCurrentState(); int line = span.Snapshot.GetLineNumberFromPosition(Math.Max(token.StartIndex, token.StopIndex)); lineStateChanged = classifierState._lineStates[line].MultilineToken || !_stateComparer.Equals(classifierState._lineStates[line].EndLineState, stateAtEndOfLine); // even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated. SetLineState(classifierState, line, new LineStateInfo(stateAtEndOfLine)); if (lineStateChanged) { if (line < span.Snapshot.LineCount - 1) { /* update the span's end position or the line state change won't be reflected * in the editor */ int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak; if (endPosition > extendedSpan.End) { spanExtended = true; extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition)); } } } } if (token.Type == IntStreamConstants.Eof) { break; } if (token.StartIndex >= span.End.Position) { break; } previousToken = token; previousTokenEndsLine = tokenEndsLine; if (token.StopIndex < requestedSpan.Start) { continue; } var tokenClassificationSpans = GetClassificationSpansForToken(token, span.Snapshot); if (tokenClassificationSpans != null) { classificationSpans.AddRange(tokenClassificationSpans); } if (!inBounds) { break; } } } if (extendMultilineSpanToLine > 0) { int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length; if (endPosition > extendedSpan.End) { spanExtended = true; extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition)); } } if (spanExtended) { /* Subtract 1 from each of these because the spans include the line break on their last * line, forcing it to appear as the first position on the following line. */ int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End); int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1; // when considering the last line of a document, span and extendedSpan may end on the same line ForceReclassifyLines(classifierState, firstLine, Math.Max(firstLine, lastLine)); } return(classificationSpans); }
public virtual IList <ClassificationSpan> GetClassificationSpans(SnapshotSpan span) { Contract.Ensures(Contract.Result <IList <ClassificationSpan> >() != null); Span requestedSpan = span; TState startState = AdjustParseSpan(ref span); ICharStream input = CreateInputStream(span); ITokenSourceWithState <TState> lexer = CreateLexer(input, startState); List <ClassificationSpan> classificationSpans = new List <ClassificationSpan>(); IToken previousToken = null; bool previousTokenEndsLine = false; /* this is held outside the loop because only tokens which end at the end of a line * impact its value. */ bool lineStateChanged = false; int extendMultilineSpanToLine = 0; SnapshotSpan extendedSpan = span; bool spanExtended = false; while (true) { IToken token = lexer.NextToken(); bool inBounds = token.StartIndex < span.End.Position; int startLineCurrent; if (token.Type == CharStreamConstants.EndOfFile) { startLineCurrent = span.Snapshot.LineCount; } else { startLineCurrent = token.Line; } if (previousToken == null || previousToken.Line < startLineCurrent - 1) { // endLinePrevious is the line number the previous token ended on int endLinePrevious; if (previousToken != null) { endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex + 1); } else { endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1; } if (startLineCurrent > endLinePrevious + 1) { int firstMultilineLine = endLinePrevious; if (previousToken == null || previousTokenEndsLine) { firstMultilineLine++; } for (int i = firstMultilineLine; i < startLineCurrent; i++) { if (!_lineStates[i].MultilineToken || lineStateChanged) { extendMultilineSpanToLine = i + 1; } if (inBounds) { SetLineState(i, LineStateInfo.Multiline); } } } } if (token.Type == CharStreamConstants.EndOfFile) { break; } previousToken = token; previousTokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token); if (IsMultilineToken(span.Snapshot, lexer, token)) { int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex); int stopLine = span.Snapshot.GetLineNumberFromPosition(token.StopIndex + 1); for (int i = startLine; i < stopLine; i++) { if (!_lineStates[i].MultilineToken) { extendMultilineSpanToLine = i + 1; } if (inBounds) { SetLineState(i, LineStateInfo.Multiline); } } } bool tokenEndsLine = previousTokenEndsLine; if (tokenEndsLine) { TState stateAtEndOfLine = lexer.GetCurrentState(); int line = span.Snapshot.GetLineNumberFromPosition(token.StopIndex + 1); lineStateChanged = _lineStates[line].MultilineToken || !_stateComparer.Equals(_lineStates[line].EndLineState, stateAtEndOfLine); // even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated. if (inBounds) { SetLineState(line, new LineStateInfo(stateAtEndOfLine)); } if (lineStateChanged) { if (line < span.Snapshot.LineCount - 1) { /* update the span's end position or the line state change won't be reflected * in the editor */ int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak; if (endPosition > extendedSpan.End) { spanExtended = true; extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition)); } } } } if (token.StartIndex >= span.End.Position) { break; } if (token.StopIndex < requestedSpan.Start) { continue; } var tokenClassificationSpans = GetClassificationSpansForToken(token, span.Snapshot); if (tokenClassificationSpans != null) { classificationSpans.AddRange(tokenClassificationSpans); } if (!inBounds) { break; } } if (extendMultilineSpanToLine > 0) { int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length; if (endPosition > extendedSpan.End) { spanExtended = true; extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition)); } } if (spanExtended) { /* Subtract 1 from each of these because the spans include the line break on their last * line, forcing it to appear as the first position on the following line. */ int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End); int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1; ForceReclassifyLines(firstLine, lastLine); } return(classificationSpans); }