public Token Peek()
{
if (tok != null)
return tok;
tok = Get();
return tok;
}
public void Unexpected(Token unexpected)
{
throw new FormatException(string.Format("Unpexected token '{0}'.", unexpected));
}
public Token Get()
{
if (tok != null)
{
var t = tok;
tok = null;
return t;
}
var st = State.Initial;
for (; ; )
{
int c = rdr.Peek();
char ch = (char) c;
switch (st)
{
case State.Initial:
switch (c)
{
case -1:
return new Token(TokenType.EOF, lineNumber);
case ' ':
case '\t':
case '\r':
st = Transition(State.Initial); break;
case '\n':
++lineNumber;
return EatTokenize(TokenType.EOL);
case ';': st = Transition(State.Comment); break;
case '=': return EatTokenize(TokenType.Eq);
case '+': return EatTokenize(TokenType.Plus);
case ':': return EatTokenize(TokenType.Colon);
case ',': return EatTokenize(TokenType.Comma);
case '#': return EatTokenize(TokenType.Hash);
case '@': return EatTokenize(TokenType.At);
case '-': return EatTokenize(TokenType.Minus);
case '(': return EatTokenize(TokenType.LParen);
case ')': return EatTokenize(TokenType.RParen);
case '.': st = StartToken(ch, State.Id); break;
default:
if (Char.IsLetter(ch))
{
st = StartToken(ch, State.Id); break;
}
else if (Char.IsDigit(ch))
{
st = StartToken(ch, State.Number); break;
}
Unexpected(c);
break;
}
break;
case State.Comment:
switch (c)
{
case -1: return new Token(TokenType.EOF, lineNumber);
case '\n': return EatTokenize(TokenType.EOL);
default: st = Transition(State.Comment); break;
}
break;
case State.Id:
switch (c)
{
case -1: return TokenizeId();
default:
if (char.IsLetterOrDigit(ch) || ch == '_')
{
st = Accumulate(ch, State.Id);
}
else
{
return TokenizeId();
}
break;
}
break;
case State.Number:
switch (c)
{
case -1: return new Token(TokenType.Number, Convert.ToInt32(sb.ToString(), 8), lineNumber);
case '.': return EatTokenize(TokenType.Number, Convert.ToInt32(sb.ToString(), 10));
default:
if (char.IsDigit(ch))
st = Accumulate(ch, State.Number);
else
return new Token(TokenType.Number, Convert.ToInt32(sb.ToString(), 8), lineNumber);
break;
}
break;
default:
throw new NotImplementedException();
}
}
}