private void HandleToken(Token t) { switch(t.type) { case TokenType.EMPTY: break; default: Kerbulator.Debug(" "+ t.ToString()); tokens.Enqueue(t); break; } }
public void Tokenize(string line) { int lineno = 1; int col = 1; Token tok = new Token(functionName, lineno, col); for(int i=0; i<line.Length; i++, col++) { char c = line[i]; switch(c) { // Whitespace case ' ': case '\t': case '\r': HandleToken(tok); tok = new Token(functionName, lineno, col); break; // End of statement case '\n': HandleToken(tok); HandleToken(new Token(TokenType.END, "\n", functionName, lineno, col)); lineno ++; tok = new Token(functionName, lineno, col); break; // Comments case '#': HandleToken(tok); // Skip to next newline int newI = line.IndexOf('\n', i+1) - 1; if(newI < 0) newI = line.Length-1; col += newI - i; i = newI; tok = new Token(functionName, lineno, col); break; // Quoted strings case '"': HandleToken(tok); tok = new Token(TokenType.TEXT, "", functionName, lineno, col); // Read until next unescaped " bool terminated = false; for(i=i+1, col++; i<line.Length; i++, col++) { if(line[i] == '\\') { if(i >= line.Length - 1) break; tok.val += line[i+1]; i += 2; col += 2; } else if(line[i] == '"') { terminated = true; break; } else { tok.val += line[i]; } } if(!terminated) throw new Exception(tok.pos + "missing end quote (\") for string"); HandleToken(tok); tok = new Token(functionName, lineno, col); break; // Numbers case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': if(tok.type == TokenType.EMPTY) { tok.type = TokenType.NUMBER; tok.val = c.ToString(); } else if(tok.type == TokenType.NUMBER || tok.type == TokenType.IDENTIFIER) tok.val += c; else throw new Exception(tok.pos + c +" is invalid here"); break; case 'e': case 'E': if(tok.type == TokenType.EMPTY) { tok.type = TokenType.IDENTIFIER; tok.val = c.ToString(); } else if(tok.type == TokenType.NUMBER || tok.type == TokenType.IDENTIFIER) { tok.val += c; if(i < line.Length - 1 && line[i+1] == '-') { tok.val += line[i+1].ToString(); i++; col++; } } else throw new Exception(tok.pos + c +" is invalid here"); break; case '.': if(tok.type == TokenType.EMPTY) { tok.type = TokenType.NUMBER; tok.val = c.ToString(); } else if(tok.type == TokenType.NUMBER) { if(tok.val.IndexOf(c) >= 0) throw new Exception(tok.pos +"numbers can only have one decimal point (.) in them"); tok.val += c; } else if(tok.type == TokenType.IDENTIFIER) { tok.val += c; } else throw new Exception(tok.pos +"variable/function names cannot start with a dot (.)"); break; // Operators case '-': case '+': case '/': case '÷': case '√': case '%': case '*': case '·': case '×': case '^': case '=': HandleToken(tok); HandleToken(new Token(TokenType.OPERATOR, c.ToString(), functionName, lineno, col)); tok = new Token(functionName, lineno, col); break; // Brackets case '[': case ']': HandleToken(tok); HandleToken(new Token(TokenType.LIST, c.ToString(), functionName, lineno, col)); tok = new Token(functionName, lineno, col); break; case '(': case ')': case '{': case '}': case '⌊': case '⌋': case '⌈': case '⌉': case '|': HandleToken(tok); HandleToken(new Token(TokenType.BRACE, c.ToString(), functionName, lineno, col)); tok = new Token(functionName, lineno, col); break; // In: and Out: statements case ':': if(tok.val == "in") HandleToken(new Token(TokenType.IN, tok.val, functionName, lineno, col)); else if(tok.val == "out") HandleToken(new Token(TokenType.OUT, tok.val, functionName, lineno, col)); else throw new Exception(tok.pos +"expected in: or out:, but encountered "+ tok.val +":"); tok = new Token(functionName, lineno, col); break; case ',': HandleToken(tok); HandleToken(new Token(TokenType.COMMA, c.ToString(), functionName, lineno, col)); tok = new Token(functionName, lineno, col); break; default: if(tok.type != TokenType.EMPTY && tok.type != TokenType.IDENTIFIER) throw new Exception(tok.pos + "unexpected character: "+ c); tok.type = TokenType.IDENTIFIER; tok.val += c; break; } } HandleToken(tok); Kerbulator.Debug("\n"); }