/// <summary>
/// Peek one token ahead of the current position
/// </summary>
/// <param name="token">The peeked token</param>
/// <returns>True if there is one more token after the current position, otherwise false</returns>
public bool Peek(out LexerToken token)
{
// read but don't advance
// implementation: actually advance the lexer
// but make sure Peek/Next call returns the same token
if (this.peekToken != null)
{
// we've already peeked
token = this.peekToken;
return(true);
}
// store current token
LexerToken temp = this.currentToken;
// advance parser
if (this.Next())
{
this.peekToken = this.currentToken;
token = this.currentToken;
// restore current state
this.currentToken = temp;
return(true);
}
else
{
this.peekToken = null;
token = null;
// restore current state
this.currentToken = temp;
return(false);
}
}
/// <summary>
/// Move to the next token
/// </summary>
/// <returns>True if lexer has moved, otherwise false</returns>
public bool Next()
{
// DEVNOTE(pqian):
// This function is called a very large number of times during text parsing.
// For example, experiments show that during simple WKT parsing, this function accounts for over 70%
// of the workload. Thus, efforts should be taken to keep this function as light weight as possible.
// Token Accumulation Logic:
// we'll accumulate whenever we are:
// 1a. starting fresh, hence currentType is null.
// 1b. we are inside a token that's building, and the new character is still part of the same token.
// we should break out of the loop if:
// 2a. MatchTokenType tell us to (terminate = true)
// 2b. We are inside a token and we've encountered a new token (currentType and peek type is different)
// It follows that most of the time we will either accumulate the token or break out of the loop
// to return the token, except in one case, where we are starting fresh AND the first character is a terminal
// character (delimeter). In this case we will both accumulate that char and break out of the loop.
if (this.peekToken != null)
{
this.currentToken = this.peekToken;
this.peekToken = null;
return(true);
}
LexerToken originalToken = this.CurrentToken;
int? currentTokenType = null;
int textValue;
StringBuilder nextTokenText = null;
bool isDelimiter = false;
while (!isDelimiter && (textValue = this.reader.Peek()) >= 0)
{
char currentChar = (char)textValue;
int newTokenType;
isDelimiter = this.MatchTokenType(currentChar, currentTokenType, out newTokenType);
if (!currentTokenType.HasValue)
{
// fresh token
currentTokenType = newTokenType;
nextTokenText = new StringBuilder();
nextTokenText.Append(currentChar);
this.reader.Read();
}
else
{
// existing token
if (currentTokenType == newTokenType)
{
// continuation of the current token
nextTokenText.Append(currentChar);
this.reader.Read();
}
else
{
// starting a new token
isDelimiter = true;
}
}
}
// we got here due to end of stream, could still have unprocessed tokens
if (currentTokenType.HasValue)
{
Debug.Assert(nextTokenText != null, "Token text should not be null if current Token type has value");
this.currentToken = new LexerToken()
{
Text = nextTokenText.ToString(), Type = currentTokenType.Value
};
}
return(originalToken != this.currentToken);
}
