Exemple #1
0
 public Lexer(System.IO.TextReader rdr)
 {
     this.rdr = rdr;
     st = State.StartOfLine;
     lookahead = new Token(TokenType.EOFile);
     sb = new StringBuilder();
 }
Exemple #2
0
 public Token PeekToken()
 {
     if (lookahead.Type == TokenType.EOFile)
     {
         lookahead = ReadToken();
     }
     return lookahead;
 }
Exemple #3
0
 public Token GetToken()
 {
     if (lookahead.Type != TokenType.EOFile)
     {
         var t = lookahead;
         lookahead = new Token(TokenType.EOFile);
         return t;
     }
     return ReadToken();
 }
Exemple #4
0
 private Token BuildToken(TokenType tokenType, State nextState)
 {
     var t = new Token(tokenType, sb.ToString());
     Debug.Write(string.Format("{0} {1} ", t.Type, t.Text));
     sb = new StringBuilder();
     st = nextState;
     return t;
 }