private void HandleToken(Token t)
{
switch(t.type) {
case TokenType.EMPTY:
break;
default:
Kerbulator.Debug(" "+ t.ToString());
tokens.Enqueue(t);
break;
}
}
public void Tokenize(string line)
{
int lineno = 1;
int col = 1;
Token tok = new Token(functionName, lineno, col);
for(int i=0; i<line.Length; i++, col++) {
char c = line[i];
switch(c)
{
// Whitespace
case ' ':
case '\t':
case '\r':
HandleToken(tok);
tok = new Token(functionName, lineno, col);
break;
// End of statement
case '\n':
HandleToken(tok);
HandleToken(new Token(TokenType.END, "\n", functionName, lineno, col));
lineno ++;
tok = new Token(functionName, lineno, col);
break;
// Comments
case '#':
HandleToken(tok);
// Skip to next newline
int newI = line.IndexOf('\n', i+1) - 1;
if(newI < 0)
newI = line.Length-1;
col += newI - i;
i = newI;
tok = new Token(functionName, lineno, col);
break;
// Quoted strings
case '"':
HandleToken(tok);
tok = new Token(TokenType.TEXT, "", functionName, lineno, col);
// Read until next unescaped "
bool terminated = false;
for(i=i+1, col++; i<line.Length; i++, col++) {
if(line[i] == '\\') {
if(i >= line.Length - 1)
break;
tok.val += line[i+1];
i += 2; col += 2;
} else if(line[i] == '"') {
terminated = true;
break;
} else {
tok.val += line[i];
}
}
if(!terminated)
throw new Exception(tok.pos + "missing end quote (\") for string");
HandleToken(tok);
tok = new Token(functionName, lineno, col);
break;
// Numbers
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
if(tok.type == TokenType.EMPTY) {
tok.type = TokenType.NUMBER;
tok.val = c.ToString();
} else if(tok.type == TokenType.NUMBER || tok.type == TokenType.IDENTIFIER)
tok.val += c;
else
throw new Exception(tok.pos + c +" is invalid here");
break;
case 'e':
case 'E':
if(tok.type == TokenType.EMPTY) {
tok.type = TokenType.IDENTIFIER;
tok.val = c.ToString();
} else if(tok.type == TokenType.NUMBER || tok.type == TokenType.IDENTIFIER) {
tok.val += c;
if(i < line.Length - 1 && line[i+1] == '-') {
tok.val += line[i+1].ToString();
i++; col++;
}
} else
throw new Exception(tok.pos + c +" is invalid here");
break;
case '.':
if(tok.type == TokenType.EMPTY) {
tok.type = TokenType.NUMBER;
tok.val = c.ToString();
} else if(tok.type == TokenType.NUMBER) {
if(tok.val.IndexOf(c) >= 0)
throw new Exception(tok.pos +"numbers can only have one decimal point (.) in them");
tok.val += c;
} else if(tok.type == TokenType.IDENTIFIER) {
tok.val += c;
} else
throw new Exception(tok.pos +"variable/function names cannot start with a dot (.)");
break;
// Operators
case '-':
case '+':
case '/':
case '÷':
case '√':
case '%':
case '*':
case '·':
case '×':
case '^':
case '=':
HandleToken(tok);
HandleToken(new Token(TokenType.OPERATOR, c.ToString(), functionName, lineno, col));
tok = new Token(functionName, lineno, col);
break;
// Brackets
case '[':
case ']':
HandleToken(tok);
HandleToken(new Token(TokenType.LIST, c.ToString(), functionName, lineno, col));
tok = new Token(functionName, lineno, col);
break;
case '(':
case ')':
case '{':
case '}':
case '⌊':
case '⌋':
case '⌈':
case '⌉':
case '|':
HandleToken(tok);
HandleToken(new Token(TokenType.BRACE, c.ToString(), functionName, lineno, col));
tok = new Token(functionName, lineno, col);
break;
// In: and Out: statements
case ':':
if(tok.val == "in")
HandleToken(new Token(TokenType.IN, tok.val, functionName, lineno, col));
else if(tok.val == "out")
HandleToken(new Token(TokenType.OUT, tok.val, functionName, lineno, col));
else
throw new Exception(tok.pos +"expected in: or out:, but encountered "+ tok.val +":");
tok = new Token(functionName, lineno, col);
break;
case ',':
HandleToken(tok);
HandleToken(new Token(TokenType.COMMA, c.ToString(), functionName, lineno, col));
tok = new Token(functionName, lineno, col);
break;
default:
if(tok.type != TokenType.EMPTY && tok.type != TokenType.IDENTIFIER)
throw new Exception(tok.pos + "unexpected character: "+ c);
tok.type = TokenType.IDENTIFIER;
tok.val += c;
break;
}
}
HandleToken(tok);
Kerbulator.Debug("\n");
}