TokenType checkKeyword(token.Token token, int tokenOffset, string chkRest, TokenType type)
{
int chkLength = chkRest.Length;
if (token.Length != chkLength + tokenOffset)
{
return(TokenType.IDENT);
}
int offset = token.Start + tokenOffset;
for (int i = 0; i < chkLength; i++)
{
if (input[offset + i] != chkRest[i])
{
return(TokenType.IDENT);
}
}
return(type);
}
/*
* Originally in the Token class
*/
TokenType LookupIdent(token.Token token)
{
/* KEYWORDS
*
* else
* exit
* false
* fn
* if
* let
* return
* true
*
*/
int tokenStart = token.Start;
if (token.Length == 1) // no single char keywords
{
return(TokenType.IDENT);
}
// clox's approach
switch (input[tokenStart])
{
case 'e':
switch (input[tokenStart + 1])
{
case 'l':
return(checkKeyword(token, 2, "se", TokenType.ELSE));
case 'x':
return(checkKeyword(token, 2, "it", TokenType.EXIT));
}
break;
case 'f':
switch (input[tokenStart + 1])
{
case 'a':
return(checkKeyword(token, 2, "lse", TokenType.FALSE));
case 'n':
return(checkKeyword(token, 2, "", TokenType.FUNCTION));
}
break;
case 'i':
return(checkKeyword(token, 1, "f", TokenType.IF));
case 'l':
return(checkKeyword(token, 1, "et", TokenType.LET));
case 'r':
return(checkKeyword(token, 1, "eturn", TokenType.RETURN));
case 't':
return(checkKeyword(token, 1, "rue", TokenType.TRUE));
}
return(TokenType.IDENT);
}
static void nextToken()
{
curToken = peekToken;
peekToken = l.NextToken();
}
public token.Token NextToken()
{
token.Token tok = new token.Token();
this.skipWhitespace();
skipSoloSemiColons();
switch (this.ch)
{
case '=':
if (this.peekChar() == '=')
{
this.readChar();
tok = newToken(TokenType.EQ, 2);
}
else
{
tok = newToken(TokenType.ASSIGN, 1);
}
break;
case '+':
tok = newToken(TokenType.PLUS, 1);
break;
case '-':
tok = newToken(TokenType.MINUS, 1);
break;
case '!':
if (this.peekChar() == '=')
{
this.readChar();
tok = newToken(TokenType.NOT_EQ, 2);
}
else
{
tok = newToken(TokenType.BANG, 1);
}
break;
case '/':
tok = newToken(TokenType.SLASH, 1);
break;
case '*':
tok = newToken(TokenType.ASTERISK, 1);
break;
case '<':
if (this.peekChar() == '=')
{
this.readChar();
tok = newToken(TokenType.LT_EQ, 2);
}
else if (this.peekChar() == '<')
{
this.readChar();
tok = newToken(TokenType.SHL, 2);
}
else
{
tok = newToken(TokenType.LT, 1);
}
break;
case '>':
if (this.peekChar() == '=')
{
this.readChar();
tok = newToken(TokenType.GT_EQ, 2);
}
else if (this.peekChar() == '>')
{
this.readChar();
tok = newToken(TokenType.SHR, 2);
}
else
{
tok = newToken(TokenType.GT, 1);
}
break;
case '^':
tok = newToken(TokenType.CARET, 1);
break;
case '%':
tok = newToken(TokenType.PERCENT, 1);
break;
case ';':
tok = newToken(TokenType.SEMICOLON, 1);
break;
case ':':
tok = newToken(TokenType.COLON, 1);
break;
case ',':
tok = newToken(TokenType.COMMA, 1);
break;
case '{':
tok = newToken(TokenType.LBRACE, 1);
break;
case '}':
tok = newToken(TokenType.RBRACE, 1);
break;
case '(':
tok = newToken(TokenType.LPAREN, 1);
break;
case ')':
tok = newToken(TokenType.RPAREN, 1);
break;
case '"':
tok.Type = TokenType.STRING;
tok.Start = position + 1;
tok.Length = this.readString();
break;
case '[':
tok = newToken(TokenType.LBRACKET, 1);
break;
case ']':
tok = newToken(TokenType.RBRACKET, 1);
break;
case '\0':
tok.Length = 0;
tok.Type = TokenType.EOF;
break;
default:
tok.Start = position;
if (isLetter(this.ch))
{
tok.Length = this.readIdentifier();
tok.Type = this.LookupIdent(tok);
lastTokenType = tok.Type;
return(tok);
}
else if (isDigit(this.ch))
{
tok.Type = TokenType.INT;
tok.Length = this.readNumber();
lastTokenType = tok.Type;
return(tok);
}
else
{
tok = newToken(TokenType.ILLEGAL, 1);
}
break;
}
this.readChar();
lastTokenType = tok.Type;
return(tok);
}
public token.Token NextToken()
{
token.Token tok = new token.Token();
this.skipWhitespace();
switch (this.ch)
{
case '=':
if (this.peekChar() == '=')
{
char ch = this.ch;
this.readChar();
string literal = ch.ToString() + this.ch.ToString();
tok = new token.Token {
Type = token.EQ, Literal = literal
};
}
else
{
tok = newToken(token.ASSIGN, this.ch);
}
break;
case '+':
tok = newToken(token.PLUS, this.ch);
break;
case '-':
tok = newToken(token.MINUS, this.ch);
break;
case '!':
if (this.peekChar() == '=')
{
char ch = this.ch;
this.readChar();
string literal = ch.ToString() + this.ch.ToString();
tok = new token.Token {
Type = token.NOT_EQ, Literal = literal
};
}
else
{
tok = newToken(token.BANG, this.ch);
}
break;
case '/':
tok = newToken(token.SLASH, this.ch);
break;
case '*':
tok = newToken(token.ASTERISK, this.ch);
break;
case '<':
tok = newToken(token.LT, this.ch);
break;
case '>':
tok = newToken(token.GT, this.ch);
break;
case ';':
tok = newToken(token.SEMICOLON, this.ch);
break;
case ':':
tok = newToken(token.COLON, this.ch);
break;
case ',':
tok = newToken(token.COMMA, this.ch);
break;
case '{':
tok = newToken(token.LBRACE, this.ch);
break;
case '}':
tok = newToken(token.RBRACE, this.ch);
break;
case '(':
tok = newToken(token.LPAREN, this.ch);
break;
case ')':
tok = newToken(token.RPAREN, this.ch);
break;
case '"':
tok.Type = token.STRING;
tok.Literal = this.readString();
break;
case '[':
tok = newToken(token.LBRACKET, this.ch);
break;
case ']':
tok = newToken(token.RBRACKET, this.ch);
break;
case '\0':
tok.Literal = "";
tok.Type = token.EOF;
break;
default:
if (isLetter(this.ch))
{
tok.Literal = this.readIdentifier();
tok.Type = token.LookupIdent(tok.Literal);
return(tok);
}
else if (isDigit(this.ch))
{
tok.Type = token.INT;
tok.Literal = this.readNumber();
return(tok);
}
else
{
tok = newToken(token.ILLEGAL, this.ch);
}
break;
}
this.readChar();
return(tok);
}
