public IToken NextToken()
{
var next = _source.NextToken();
//collect the tokens, but don't send them to parser
while (next.Type == QuestScriptLexer.Comment ||
next.Type == QuestScriptLexer.LineComment)
{
_commentTokens.Add(next);
next = _source.NextToken();
}
return(next);
}
/** <summary>
* Load all tokens from the token source and put in tokens.
* This is done upon first LT request because you might want to
* set some token type / channel overrides before filling buffer.
* </summary>
*/
public virtual void FillBuffer()
{
// fast return if the buffer is already full
if (p != -1)
{
return;
}
int index = 0;
IToken t = _tokenSource.NextToken();
while (t != null && t.Type != CharStreamConstants.EndOfFile)
{
bool discard = false;
// is there a channel override for token type?
int channelI;
if (channelOverrideMap != null && channelOverrideMap.TryGetValue(t.Type, out channelI))
{
t.Channel = channelI;
}
//if ( channelOverrideMap != null && channelOverrideMap.ContainsKey( t.getType() ) )
//{
// object channelI = channelOverrideMap.get( t.getType() );
// if ( channelI != null )
// {
// t.setChannel( (int)channelI );
// }
//}
if (discardSet != null &&
discardSet.Contains(t.Type))
{
discard = true;
}
else if (discardOffChannelTokens && t.Channel != this.channel)
{
discard = true;
}
if (!discard)
{
t.TokenIndex = index;
tokens.Add(t);
index++;
}
t = _tokenSource.NextToken();
}
// leave p pointing at first token on channel
p = 0;
p = SkipOffTokenChannels(p);
}
private IToken NextTokenImpl()
{
if (_nextTokenIsSemicolon)
{
_nextTokenIsSemicolon = false;
return
(new CommonToken(_nextRealToken.InputStream, GoLexer.SEMI, TokenChannels.Default, _nextRealToken.StartIndex, _nextRealToken.StopIndex)
{
Text = ";*"
});
}
// handle the first token in the stream
if (_nextRealToken == null)
{
_nextRealToken = _lexer.NextToken();
}
IToken token = _nextRealToken;
_nextRealToken = _lexer.NextToken();
if (_nextRealToken.Type == GoLexer.NEW_LINE)
{
switch (token.Type)
{
case GoLexer.IDENTIFIER:
case GoLexer.NUMBER:
case GoLexer.CHAR_LITERAL:
case GoLexer.STRING_LITERAL:
case GoLexer.KW_BREAK:
case GoLexer.KW_CONTINUE:
case GoLexer.KW_FALLTHROUGH:
case GoLexer.KW_RETURN:
case GoLexer.INC:
case GoLexer.DEC:
case GoLexer.RPAREN:
case GoLexer.RBRACK:
case GoLexer.RBRACE:
_nextTokenIsSemicolon = true;
break;
default:
break;
}
}
return(token);
}
/// <summary>
/// Add
/// <paramref name="n"/>
/// elements to the buffer. Returns the number of tokens
/// actually added to the buffer. If the return value is less than
/// <paramref name="n"/>
/// ,
/// then EOF was reached before
/// <paramref name="n"/>
/// tokens could be added.
/// </summary>
protected internal virtual int Fill(int n)
{
for (int i = 0; i < n; i++)
{
if (this.n > 0 && tokens[this.n - 1].Type == TokenConstants.EOF)
{
return(i);
}
IToken t = TokenSource.NextToken();
Add(t);
}
return(n);
}
/** <summary>
* Load all tokens from the token source and put in tokens.
* This is done upon first LT request because you might want to
* set some token type / channel overrides before filling buffer.
* </summary>
*/
public void FillBuffer()
{
// fast return if the buffer is already full
if (p != -1)
{
return;
}
int index = 0;
SlimToken t = _tokenSource.NextToken();
while (t.Type != CharStreamConstants.EndOfFile)
{
//t.TokenIndex = index;
tokens.Add(t);
index++;
t = _tokenSource.NextToken();
}
// leave p pointing at first token on channel
p = 0;
p = SkipOffTokenChannels(p);
}
/// <summary>Load all tokens from the token source and put in tokens.
/// This is done upon first LT request because you might want to
/// set some token type / channel overrides before filling buffer.
/// </summary>
protected virtual void FillBuffer()
{
int index = 0;
IToken t = tokenSource.NextToken();
while ((t != null) && (t.Type != (int)Antlr.Runtime.CharStreamConstants.EOF))
{
bool discard = false;
// is there a channel override for token type?
if (channelOverrideMap != null)
{
object channelI = channelOverrideMap[(int)t.Type];
if (channelI != null)
{
t.Channel = (int)channelI;
}
}
if (discardSet != null && discardSet.Contains(t.Type.ToString()))
{
discard = true;
}
else if (discardOffChannelTokens && t.Channel != this.channel)
{
discard = true;
}
if (!discard)
{
t.TokenIndex = index;
tokens.Add(t);
index++;
}
t = tokenSource.NextToken();
}
// leave p pointing at first token on channel
p = 0;
p = SkipOffTokenChannels(p);
}
/// <summary>
/// Add
/// <paramref name="n"/>
/// elements to buffer.
/// </summary>
/// <returns>The actual number of elements added to the buffer.</returns>
protected internal virtual int Fetch(int n)
{
if (fetchedEOF)
{
return(0);
}
for (int i = 0; i < n; i++)
{
IToken t = _tokenSource.NextToken();
if (t is IWritableToken)
{
((IWritableToken)t).TokenIndex = tokens.Count;
}
tokens.Add(t);
if (t.Type == TokenConstants.Eof)
{
fetchedEOF = true;
return(i + 1);
}
}
return(n);
}
public IToken NextToken()
{
if (_caretToken == null)
{
IToken token = _source.NextToken();
if (token.StopIndex + 1 < _caretOffset)
{
// the caret is after this token; nothing special to do
}
else if (token.StartIndex > _caretOffset)
{
// the token is after the caret; no need to include it
token = new CaretToken(_tokenFactorySourcePair, TokenConstants.DefaultChannel, _caretOffset, _caretOffset);
_caretToken = token;
}
else
{
if (token.StopIndex + 1 == _caretOffset && token.StopIndex >= token.StartIndex)
{
if (!IsWordToken(token))
{
// the caret is at the end of this token, and this isn't a word token or a zero-length token
return(token);
}
}
// the caret is in the middle of or at the end of this token
token = new CaretToken(token);
_caretToken = token;
}
return(token);
}
throw new InvalidOperationException("Attempted to look past the caret.");
}
public IToken NextToken()
{
IToken token = source.NextToken();
if (token.Type == InferenceRules_ENLexer.NEWLINE)
{
string statement = statementBuffer.ToString();
if (definitions.Contains(statement))
{
htmlBuffer.Append("<font color='#006600'>");
}
else
{
htmlBuffer.Append("<font color='#FF0000'>");
}
htmlBuffer.Append(statement).Append("</font>");
htmlBuffer.Append("<br/>");
statementBuffer = new StringBuilder();
}
else if (token.Type == InferenceRules_ENLexer.TAB)
{
htmlBuffer.Append(" ");
}
else
{
if ((token.Type == InferenceRules_ENLexer.RULE) ||
(token.Type == InferenceRules_ENLexer.FACT) ||
(token.Type == InferenceRules_ENLexer.QUERY))
{
htmlBuffer.Append("<br/>");
}
if ((token.Type == InferenceRules_ENLexer.QUOTE) && (!inQuote))
{
htmlBuffer.Append("<font color='#0000FF'>");
htmlBuffer.Append(token.Text);
inQuote = true;
}
else if ((token.Type == InferenceRules_ENLexer.QUOTE) && (inQuote))
{
htmlBuffer.Append(token.Text);
htmlBuffer.Append("</font>");
inQuote = false;
}
else if (inQuote)
{
htmlBuffer.Append(token.Text);
}
else if ((token.Type == InferenceRules_ENLexer.CHAR) ||
(token.Type == InferenceRules_ENLexer.SPACE) ||
(token.Type == InferenceRules_ENLexer.NUMERIC))
{
statementBuffer.Append(token.Text);
}
else if ((token.Type == InferenceRules_ENLexer.COUNT) ||
(token.Type == InferenceRules_ENLexer.DEDUCT) ||
(token.Type == InferenceRules_ENLexer.FORGET) ||
(token.Type == InferenceRules_ENLexer.MODIFY))
{
htmlBuffer.Append("<font color='#990066'><b>").Append(token.Text).Append(" ").Append("</b></font>");
}
else
{
htmlBuffer.Append("<b>").Append(token.Text).Append(" ").Append("</b>");
}
}
return(token);
}
public IToken NextToken()
{
return(tokens.NextToken());
}
public virtual IList <ClassificationSpan> GetClassificationSpans(SnapshotSpan span)
{
Span requestedSpan = span;
AdjustParseSpan(ref span);
ICharStream input = CreateInputStream(span);
ITokenSource lexer = CreateLexer(input);
List <ClassificationSpan> classificationSpans = new List <ClassificationSpan>();
IToken previousToken = null;
bool previousTokenEndsLine = false;
/* this is held outside the loop because only tokens which end at the end of a line
* impact its value.
*/
bool lineStateChanged = false;
int extendMultilineSpanToLine = 0;
SnapshotSpan extendedSpan = span;
bool spanExtended = false;
while (true)
{
IToken token = lexer.NextToken();
bool inBounds = token.StartIndex < span.End.Position;
int startLineCurrent;
if (token.Type == CharStreamConstants.EndOfFile)
{
startLineCurrent = span.Snapshot.LineCount;
}
else
{
startLineCurrent = token.Line;
}
if (previousToken == null || previousToken.Line < startLineCurrent - 1)
{
// endLinePrevious is the line number the previous token ended on
int endLinePrevious;
if (previousToken != null)
{
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(previousToken.StopIndex + 1);
}
else
{
endLinePrevious = span.Snapshot.GetLineNumberFromPosition(span.Start) - 1;
}
if (startLineCurrent > endLinePrevious + 1)
{
int firstMultilineLine = endLinePrevious;
if (previousToken == null || previousTokenEndsLine)
{
firstMultilineLine++;
}
for (int i = firstMultilineLine; i < startLineCurrent; i++)
{
if (!_lineStates[i].MultilineToken || lineStateChanged)
{
extendMultilineSpanToLine = i + 1;
}
if (inBounds)
{
SetLineState(i, LineStateInfo.Multiline);
}
}
}
}
if (token.Type == CharStreamConstants.EndOfFile)
{
break;
}
previousToken = token;
previousTokenEndsLine = TokenEndsAtEndOfLine(span.Snapshot, lexer, token);
if (IsMultilineToken(span.Snapshot, lexer, token))
{
int startLine = span.Snapshot.GetLineNumberFromPosition(token.StartIndex);
int stopLine = span.Snapshot.GetLineNumberFromPosition(token.StopIndex + 1);
for (int i = startLine; i < stopLine; i++)
{
if (!_lineStates[i].MultilineToken)
{
extendMultilineSpanToLine = i + 1;
}
if (inBounds)
{
SetLineState(i, LineStateInfo.Multiline);
}
}
}
bool tokenEndsLine = previousTokenEndsLine;
if (tokenEndsLine)
{
int line = span.Snapshot.GetLineNumberFromPosition(token.StopIndex + 1);
lineStateChanged = _lineStates[line].MultilineToken;
// even if the state didn't change, we call SetLineState to make sure the _first/_lastChangedLine values get updated.
if (inBounds)
{
SetLineState(line, new LineStateInfo());
}
if (lineStateChanged)
{
if (line < span.Snapshot.LineCount - 1)
{
/* update the span's end position or the line state change won't be reflected
* in the editor
*/
int endPosition = span.Snapshot.GetLineFromLineNumber(line + 1).EndIncludingLineBreak;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
}
}
if (token.StartIndex >= span.End.Position)
{
break;
}
if (token.StopIndex < requestedSpan.Start)
{
continue;
}
var tokenClassificationSpans = GetClassificationSpansForToken(token, span.Snapshot);
if (tokenClassificationSpans != null)
{
classificationSpans.AddRange(tokenClassificationSpans);
}
if (!inBounds)
{
break;
}
}
if (extendMultilineSpanToLine > 0)
{
int endPosition = extendMultilineSpanToLine < span.Snapshot.LineCount ? span.Snapshot.GetLineFromLineNumber(extendMultilineSpanToLine).EndIncludingLineBreak : span.Snapshot.Length;
if (endPosition > extendedSpan.End)
{
spanExtended = true;
extendedSpan = new SnapshotSpan(extendedSpan.Snapshot, Span.FromBounds(extendedSpan.Start, endPosition));
}
}
if (spanExtended)
{
/* Subtract 1 from each of these because the spans include the line break on their last
* line, forcing it to appear as the first position on the following line.
*/
int firstLine = extendedSpan.Snapshot.GetLineNumberFromPosition(span.End);
int lastLine = extendedSpan.Snapshot.GetLineNumberFromPosition(extendedSpan.End) - 1;
ForceReclassifyLines(firstLine, lastLine);
}
return(classificationSpans);
}
