public Lexer(System.IO.TextReader rdr)
{
this.rdr = rdr;
st = State.StartOfLine;
lookahead = new Token(TokenType.EOFile);
sb = new StringBuilder();
}
public Token PeekToken()
{
if (lookahead.Type == TokenType.EOFile)
{
lookahead = ReadToken();
}
return lookahead;
}
public Token GetToken()
{
if (lookahead.Type != TokenType.EOFile)
{
var t = lookahead;
lookahead = new Token(TokenType.EOFile);
return t;
}
return ReadToken();
}
private Token BuildToken(TokenType tokenType, State nextState)
{
var t = new Token(tokenType, sb.ToString());
Debug.Write(string.Format("{0} {1} ", t.Type, t.Text));
sb = new StringBuilder();
st = nextState;
return t;
}