/// <summary>
/// Returns the next token from the stream.
/// </summary>
/// <param name="ignoreWhitespace">indicates whether the return whitespace tokens or to ignore them.</param>
/// <returns>The next token from the stream.</returns>
public Token <CTokenType> GetNextToken(bool ignoreWhitespace = true)
{
Token <TokenType> nextToken = null;
while (this.pushedTokens.Count > 0)
{
var pushedToken = this.pushedTokens.Pop() as CToken;
if (!ignoreWhitespace || pushedToken.BaseTokenType != TokenType.WHITESPACE)
{
return(pushedToken);
}
}
nextToken = this.localTokenizer.GetNextToken(ignoreWhitespace);
if (nextToken == null)
{
return(null);
}
CToken cToken = CToken.Create(nextToken, this.prevToken);
this.prevToken = nextToken;
log.Trace("Next token read at position {0} on line {1}. Type: {2}, Value: {3}",
cToken.PositionInLine, cToken.LineNumber, cToken.Type, cToken.Value);
return(cToken);
}
public static CToken Create(Token <TokenType> token, Token <TokenType> prevToken)
{
var cSourceToken = new CToken(token);
cSourceToken.Type = MapTokenType(token, prevToken);
return(cSourceToken);
}
