Beispiel #1
0
        private void HandleToken(Token t)
        {
            switch(t.type) {
                case TokenType.EMPTY:
                    break;

                default:
                    Kerbulator.Debug(" "+ t.ToString());
                    tokens.Enqueue(t);
                    break;
            }
        }
Beispiel #2
0
        public void Tokenize(string line)
        {
            int lineno = 1;
            int col = 1;
            Token tok = new Token(functionName, lineno, col);
            for(int i=0; i<line.Length; i++, col++) {
                char c = line[i];
                switch(c)
                {
                    // Whitespace
                    case ' ':
                    case '\t':
                    case '\r':
                        HandleToken(tok);
                        tok = new Token(functionName, lineno, col);
                        break;

                    // End of statement
                    case '\n':
                        HandleToken(tok);
                        HandleToken(new Token(TokenType.END, "\n", functionName, lineno, col));
                        lineno ++;
                        tok = new Token(functionName, lineno, col);
                        break;

                    // Comments
                    case '#':
                        HandleToken(tok);
                        // Skip to next newline
                        int newI = line.IndexOf('\n', i+1) - 1;
                        if(newI < 0)
                            newI = line.Length-1;
                        col += newI - i;
                        i = newI;
                        tok = new Token(functionName, lineno, col);
                        break;

                    // Quoted strings
                    case '"':
                        HandleToken(tok);
                        tok = new Token(TokenType.TEXT, "", functionName, lineno, col);

                        // Read until next unescaped "
                        bool terminated = false;
                        for(i=i+1, col++; i<line.Length; i++, col++) {
                            if(line[i] == '\\') {
                                if(i >= line.Length - 1)
                                    break;
                                tok.val += line[i+1];
                                i += 2; col += 2;
                            } else if(line[i] == '"') {
                                terminated = true;
                                break;
                            } else {
                                tok.val += line[i];
                            }
                        }

                        if(!terminated)
                            throw new Exception(tok.pos + "missing end quote (\") for string");

                        HandleToken(tok);
                        tok = new Token(functionName, lineno, col);
                        break;

                    // Numbers
                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                        if(tok.type == TokenType.EMPTY) {
                            tok.type = TokenType.NUMBER;
                            tok.val = c.ToString();
                        } else if(tok.type == TokenType.NUMBER || tok.type == TokenType.IDENTIFIER)
                            tok.val += c;
                        else
                            throw new Exception(tok.pos + c +" is invalid here");
                        break;

                    case 'e':
                    case 'E':
                        if(tok.type == TokenType.EMPTY) {
                            tok.type = TokenType.IDENTIFIER;
                            tok.val = c.ToString();
                        } else if(tok.type == TokenType.NUMBER || tok.type == TokenType.IDENTIFIER) {
                            tok.val += c;
                            if(i < line.Length - 1 && line[i+1] == '-') {
                                tok.val += line[i+1].ToString();
                                i++; col++;
                            }
                        } else
                            throw new Exception(tok.pos + c +" is invalid here");
                        break;

                    case '.':
                        if(tok.type == TokenType.EMPTY) {
                            tok.type = TokenType.NUMBER;
                            tok.val = c.ToString();
                        } else if(tok.type == TokenType.NUMBER) {
                            if(tok.val.IndexOf(c) >= 0)
                                throw new Exception(tok.pos +"numbers can only have one decimal point (.) in them");
                            tok.val += c;
                        } else if(tok.type == TokenType.IDENTIFIER) {
                            tok.val += c;
                        } else
                            throw new Exception(tok.pos +"variable/function names cannot start with a dot (.)");
                        break;

                    // Operators
                    case '-':
                    case '+':
                    case '/':
                    case '÷':
                    case '√':
                    case '%':
                    case '*':
                    case '·':
                    case '×':
                    case '^':
                    case '=':
                        HandleToken(tok);
                        HandleToken(new Token(TokenType.OPERATOR, c.ToString(), functionName, lineno, col));
                        tok = new Token(functionName, lineno, col);
                        break;

                    // Brackets
                    case '[':
                    case ']':
                        HandleToken(tok);
                        HandleToken(new Token(TokenType.LIST, c.ToString(), functionName, lineno, col));
                        tok = new Token(functionName, lineno, col);
                        break;

                    case '(':
                    case ')':
                    case '{':
                    case '}':
                    case '⌊':
                    case '⌋':
                    case '⌈':
                    case '⌉':
                    case '|':
                        HandleToken(tok);
                        HandleToken(new Token(TokenType.BRACE, c.ToString(), functionName, lineno, col));
                        tok = new Token(functionName, lineno, col);
                        break;

                    // In: and Out: statements
                    case ':':
                        if(tok.val == "in")
                            HandleToken(new Token(TokenType.IN, tok.val, functionName, lineno, col));
                        else if(tok.val == "out")
                            HandleToken(new Token(TokenType.OUT, tok.val, functionName, lineno, col));
                        else
                            throw new Exception(tok.pos +"expected in: or out:, but encountered "+ tok.val +":");

                        tok = new Token(functionName, lineno, col);
                        break;

                    case ',':
                        HandleToken(tok);
                        HandleToken(new Token(TokenType.COMMA, c.ToString(), functionName, lineno, col));
                        tok = new Token(functionName, lineno, col);
                        break;

                    default:
                        if(tok.type != TokenType.EMPTY && tok.type != TokenType.IDENTIFIER)
                            throw new Exception(tok.pos + "unexpected character: "+ c);
                        tok.type = TokenType.IDENTIFIER;
                        tok.val += c;
                        break;
                }
            }

            HandleToken(tok);
            Kerbulator.Debug("\n");
        }