TokenType checkKeyword(token.Token token, int tokenOffset, string chkRest, TokenType type) { int chkLength = chkRest.Length; if (token.Length != chkLength + tokenOffset) { return(TokenType.IDENT); } int offset = token.Start + tokenOffset; for (int i = 0; i < chkLength; i++) { if (input[offset + i] != chkRest[i]) { return(TokenType.IDENT); } } return(type); }
/* * Originally in the Token class */ TokenType LookupIdent(token.Token token) { /* KEYWORDS * * else * exit * false * fn * if * let * return * true * */ int tokenStart = token.Start; if (token.Length == 1) // no single char keywords { return(TokenType.IDENT); } // clox's approach switch (input[tokenStart]) { case 'e': switch (input[tokenStart + 1]) { case 'l': return(checkKeyword(token, 2, "se", TokenType.ELSE)); case 'x': return(checkKeyword(token, 2, "it", TokenType.EXIT)); } break; case 'f': switch (input[tokenStart + 1]) { case 'a': return(checkKeyword(token, 2, "lse", TokenType.FALSE)); case 'n': return(checkKeyword(token, 2, "", TokenType.FUNCTION)); } break; case 'i': return(checkKeyword(token, 1, "f", TokenType.IF)); case 'l': return(checkKeyword(token, 1, "et", TokenType.LET)); case 'r': return(checkKeyword(token, 1, "eturn", TokenType.RETURN)); case 't': return(checkKeyword(token, 1, "rue", TokenType.TRUE)); } return(TokenType.IDENT); }
static void nextToken() { curToken = peekToken; peekToken = l.NextToken(); }
public token.Token NextToken() { token.Token tok = new token.Token(); this.skipWhitespace(); skipSoloSemiColons(); switch (this.ch) { case '=': if (this.peekChar() == '=') { this.readChar(); tok = newToken(TokenType.EQ, 2); } else { tok = newToken(TokenType.ASSIGN, 1); } break; case '+': tok = newToken(TokenType.PLUS, 1); break; case '-': tok = newToken(TokenType.MINUS, 1); break; case '!': if (this.peekChar() == '=') { this.readChar(); tok = newToken(TokenType.NOT_EQ, 2); } else { tok = newToken(TokenType.BANG, 1); } break; case '/': tok = newToken(TokenType.SLASH, 1); break; case '*': tok = newToken(TokenType.ASTERISK, 1); break; case '<': if (this.peekChar() == '=') { this.readChar(); tok = newToken(TokenType.LT_EQ, 2); } else if (this.peekChar() == '<') { this.readChar(); tok = newToken(TokenType.SHL, 2); } else { tok = newToken(TokenType.LT, 1); } break; case '>': if (this.peekChar() == '=') { this.readChar(); tok = newToken(TokenType.GT_EQ, 2); } else if (this.peekChar() == '>') { this.readChar(); tok = newToken(TokenType.SHR, 2); } else { tok = newToken(TokenType.GT, 1); } break; case '^': tok = newToken(TokenType.CARET, 1); break; case '%': tok = newToken(TokenType.PERCENT, 1); break; case ';': tok = newToken(TokenType.SEMICOLON, 1); break; case ':': tok = newToken(TokenType.COLON, 1); break; case ',': tok = newToken(TokenType.COMMA, 1); break; case '{': tok = newToken(TokenType.LBRACE, 1); break; case '}': tok = newToken(TokenType.RBRACE, 1); break; case '(': tok = newToken(TokenType.LPAREN, 1); break; case ')': tok = newToken(TokenType.RPAREN, 1); break; case '"': tok.Type = TokenType.STRING; tok.Start = position + 1; tok.Length = this.readString(); break; case '[': tok = newToken(TokenType.LBRACKET, 1); break; case ']': tok = newToken(TokenType.RBRACKET, 1); break; case '\0': tok.Length = 0; tok.Type = TokenType.EOF; break; default: tok.Start = position; if (isLetter(this.ch)) { tok.Length = this.readIdentifier(); tok.Type = this.LookupIdent(tok); lastTokenType = tok.Type; return(tok); } else if (isDigit(this.ch)) { tok.Type = TokenType.INT; tok.Length = this.readNumber(); lastTokenType = tok.Type; return(tok); } else { tok = newToken(TokenType.ILLEGAL, 1); } break; } this.readChar(); lastTokenType = tok.Type; return(tok); }
public token.Token NextToken() { token.Token tok = new token.Token(); this.skipWhitespace(); switch (this.ch) { case '=': if (this.peekChar() == '=') { char ch = this.ch; this.readChar(); string literal = ch.ToString() + this.ch.ToString(); tok = new token.Token { Type = token.EQ, Literal = literal }; } else { tok = newToken(token.ASSIGN, this.ch); } break; case '+': tok = newToken(token.PLUS, this.ch); break; case '-': tok = newToken(token.MINUS, this.ch); break; case '!': if (this.peekChar() == '=') { char ch = this.ch; this.readChar(); string literal = ch.ToString() + this.ch.ToString(); tok = new token.Token { Type = token.NOT_EQ, Literal = literal }; } else { tok = newToken(token.BANG, this.ch); } break; case '/': tok = newToken(token.SLASH, this.ch); break; case '*': tok = newToken(token.ASTERISK, this.ch); break; case '<': tok = newToken(token.LT, this.ch); break; case '>': tok = newToken(token.GT, this.ch); break; case ';': tok = newToken(token.SEMICOLON, this.ch); break; case ':': tok = newToken(token.COLON, this.ch); break; case ',': tok = newToken(token.COMMA, this.ch); break; case '{': tok = newToken(token.LBRACE, this.ch); break; case '}': tok = newToken(token.RBRACE, this.ch); break; case '(': tok = newToken(token.LPAREN, this.ch); break; case ')': tok = newToken(token.RPAREN, this.ch); break; case '"': tok.Type = token.STRING; tok.Literal = this.readString(); break; case '[': tok = newToken(token.LBRACKET, this.ch); break; case ']': tok = newToken(token.RBRACKET, this.ch); break; case '\0': tok.Literal = ""; tok.Type = token.EOF; break; default: if (isLetter(this.ch)) { tok.Literal = this.readIdentifier(); tok.Type = token.LookupIdent(tok.Literal); return(tok); } else if (isDigit(this.ch)) { tok.Type = token.INT; tok.Literal = this.readNumber(); return(tok); } else { tok = newToken(token.ILLEGAL, this.ch); } break; } this.readChar(); return(tok); }