Esempio n. 1
0
 public Token Peek()
 {
     if (tok != null)
         return tok;
     tok = Get();
     return tok;
 }
Esempio n. 2
0
 public void Unexpected(Token unexpected)
 {
     throw new FormatException(string.Format("Unpexected token '{0}'.", unexpected));
 }
Esempio n. 3
0
 public Token Get()
 {
     if (tok != null)
     {
         var t = tok;
         tok = null;
         return t;
     }
     var st = State.Initial;
     for (; ; )
     {
         int c = rdr.Peek();
         char ch = (char) c;
         switch (st)
         {
         case State.Initial:
             switch (c)
             {
             case -1:
                 return new Token(TokenType.EOF, lineNumber);
             case ' ':
             case '\t':
             case '\r':
                 st = Transition(State.Initial); break;
             case '\n':
                 ++lineNumber;
                 return EatTokenize(TokenType.EOL);
             case ';': st = Transition(State.Comment); break;
             case '=': return EatTokenize(TokenType.Eq);
             case '+': return EatTokenize(TokenType.Plus);
             case ':': return EatTokenize(TokenType.Colon);
             case ',': return EatTokenize(TokenType.Comma);
             case '#': return EatTokenize(TokenType.Hash);
             case '@': return EatTokenize(TokenType.At);
             case '-': return EatTokenize(TokenType.Minus);
             case '(': return EatTokenize(TokenType.LParen);
             case ')': return EatTokenize(TokenType.RParen);
             case '.': st = StartToken(ch, State.Id); break;
             default:
                 if (Char.IsLetter(ch))
                 {
                    st =  StartToken(ch, State.Id); break;
                 }
                 else if (Char.IsDigit(ch))
                 {
                     st = StartToken(ch, State.Number); break;
                 }
                 Unexpected(c);
                 break;
             }
             break;
         case State.Comment:
             switch (c)
             {
             case -1: return new Token(TokenType.EOF, lineNumber);
             case '\n': return EatTokenize(TokenType.EOL);
             default: st = Transition(State.Comment); break;
             }
             break;
         case State.Id:
             switch (c)
             {
             case -1: return TokenizeId();
             default:
                 if (char.IsLetterOrDigit(ch) || ch == '_')
                 {
                     st = Accumulate(ch, State.Id);
                 }
                 else
                 {
                     return TokenizeId();
                 }
                 break;
             }
             break;
         case State.Number:
             switch (c)
             {
             case -1: return new Token(TokenType.Number, Convert.ToInt32(sb.ToString(), 8), lineNumber);
             case '.': return EatTokenize(TokenType.Number, Convert.ToInt32(sb.ToString(), 10));
             default:
                 if (char.IsDigit(ch))
                     st = Accumulate(ch, State.Number);
                 else 
                     return new Token(TokenType.Number, Convert.ToInt32(sb.ToString(), 8), lineNumber);
                 break;
             }
             break;
         default:
             throw new NotImplementedException();
         }
     }
 }