Ejemplo n.º 1
0
        private static tok_state tok_new()
        {
            tok_state tok = PyMem_NEW_tok_state(1);

            if (tok == null)
            {
                return(null);
            }
            tok.buf            = tok.cur = tok.end = tok.inp = tok.start = null;
            tok.done           = E_OK;
            tok.fp             = null;
            tok.tabsize        = TABSIZE;
            tok.indent         = 0;
            tok.indstack[0]    = 0;
            tok.atbol          = 1;
            tok.pendin         = 0;
            tok.prompt         = tok.nextprompt = null;
            tok.lineno         = 0;
            tok.level          = 0;
            tok.filename       = null;
            tok.altwarning     = 0;
            tok.alterror       = 0;
            tok.alttabsize     = 1;
            tok.altindstack[0] = 0;
            return(tok);
        }
Ejemplo n.º 2
0
 public static void PyTokenizer_Free(tok_state tok)
 {
     if (tok.fp != null && tok.buf != null)
     {
         PyMem_DEL(tok.buf);
     }
     PyMem_DEL(ref tok);
 }
Ejemplo n.º 3
0
        public static tok_state PyTokenizer_FromString(CharPtr str)
        {
            tok_state tok = tok_new();

            if (tok == null)
            {
                return(null);
            }
            tok.buf = new CharPtr(str);
            tok.cur = new CharPtr(str);
            tok.end = new CharPtr(str);
            tok.inp = new CharPtr(str);
            return(tok);
        }
Ejemplo n.º 4
0
 private static void tok_backup(tok_state tok, int c)
 {
     if (c != EOF)
     {
         tok.cur.dec();
         if (tok.cur < tok.buf)
         {
             Py_FatalError("tok_backup: begin of buffer");
         }
         if (tok.cur[0] != c)
         {
             tok.cur[0] = (char)c;
         }
     }
 }
Ejemplo n.º 5
0
 private static int indenterror(tok_state tok)
 {
     if (0 != tok.alterror)
     {
         tok.done = E_TABSPACE;
         tok.cur  = tok.inp;
         return(1);
     }
     if (0 != tok.altwarning)
     {
         PySys_WriteStderr("%s: inconsistent use of tabs and spaces " +
                           "in indentation\n", tok.filename);
         tok.altwarning = 0;
     }
     return(0);
 }
Ejemplo n.º 6
0
        public static tok_state PyTokenizer_FromFile(FILEPtr fp, CharPtr ps1, CharPtr ps2)
        {
            tok_state tok = tok_new();

            if (tok == null)
            {
                return(null);
            }
            if ((tok.buf = PyMem_NEW_char2(BUFSIZ)) == null)
            {
                PyMem_DEL(ref tok);
                return(null);
            }
            tok.cur        = new CharPtr(tok.buf);
            tok.inp        = new CharPtr(tok.buf);
            tok.end        = new CharPtr(tok.buf, BUFSIZ);
            tok.fp         = new FILEPtr(fp);
            tok.prompt     = new CharPtr(ps1);
            tok.nextprompt = new CharPtr(ps2);
            return(tok);
        }
Ejemplo n.º 7
0
 public static void PyMem_DEL(ref tok_state x)
 {
 }
Ejemplo n.º 8
0
        private static node parsetok(tok_state tok, grammar g, int start, perrdetail err_ret, int flags)
        {
            parser_state ps;
            node         n;
            int          started = 0;

            if ((ps = PyParser_New(g, start)) == null)
            {
                fprintf(stderr, "no mem for new parser\n");
                err_ret.error = E_NOMEM;
                return(null);
            }
            if (0 != (flags & PyPARSE_YIELD_IS_KEYWORD))
            {
                ps.p_generators = 1;
            }
            for (;;)
            {
                CharPtr a = null, b = null;
                int     type;
                size_t  len;
                CharPtr str;

                type = PyTokenizer_Get(tok, ref a, ref b);
                if (type == ERRORTOKEN)
                {
                    err_ret.error = tok.done;
                    break;
                }
                if (type == ENDMARKER && 0 != started)
                {
                    type    = NEWLINE;
                    started = 0;
                }
                else
                {
                    started = 1;
                }
                len = (size_t)(b - a);
                str = PyMem_NEW_char2((int)(len + 1));
                if (str == null)
                {
                    fprintf(stderr, "no mem for next token\n");
                    err_ret.error = E_NOMEM;
                    break;
                }
                if (len > 0)
                {
                    strncpy(str, a, (int)len);
                }
                str[len] = '\0';
                if (type == NAME && 0 == ps.p_generators &&
                    len == 5 && str[0] == 'y' && strcmp(str, "yield") == 0)
                {
                    PySys_WriteStderr(yield_msg,
                                      err_ret.filename == null ?
                                      "<string>" : err_ret.filename,
                                      tok.lineno);
                }
                if ((err_ret.error =
                         PyParser_AddToken(ps, (int)type, str, tok.lineno,
                                           ref err_ret.expected)) != E_OK)
                {
                    if (err_ret.error != E_DONE)
                    {
                        PyMem_DEL(str);
                    }
                    break;
                }
            }
            if (err_ret.error == E_DONE)
            {
                n         = ps.p_tree;
                ps.p_tree = null;
            }
            else
            {
                n = null;
            }
            PyParser_Delete(ps);

            if (n == null)
            {
                if (tok.lineno <= 1 && tok.done == E_EOF)
                {
                    err_ret.error = E_EOF;
                }
                err_ret.lineno = tok.lineno;
                err_ret.offset = tok.cur - tok.buf;
                if (tok.buf != null)
                {
                    size_t len = (size_t)(tok.inp - tok.buf);
                    err_ret.text = PyMem_NEW_char2((int)(len + 1));
                    if (err_ret.text != null)
                    {
                        if (len > 0)
                        {
                            strncpy(err_ret.text, tok.buf, (int)len);
                        }
                        err_ret.text[len] = '\0';
                    }
                }
            }

            PyTokenizer_Free(tok);

            return(n);
        }
Ejemplo n.º 9
0
        public static int PyTokenizer_Get(tok_state tok, ref CharPtr p_start,
                                          ref CharPtr p_end)
        {
            int c;
            int blankline;

            p_start = p_end = null;
nextline:
            tok.start = null;
            blankline = 0;

            if (0 != tok.atbol)
            {
                int col    = 0;
                int altcol = 0;
                tok.atbol = 0;
                for (;;)
                {
                    c = tok_nextc(tok);
                    if (c == ' ')
                    {
                        col++; altcol++;
                    }
                    else if (c == '\t')
                    {
                        col    = (col / tok.tabsize + 1) * tok.tabsize;
                        altcol = (altcol / tok.alttabsize + 1)
                                 * tok.alttabsize;
                    }
                    else if (c == '\x0C')                    //'\014')
                    {
                        col = altcol = 0;
                    }
                    else
                    {
                        break;
                    }
                }
                tok_backup(tok, c);
                if (c == '#' || c == '\n')
                {
                    if (col == 0 && c == '\n' && tok.prompt != null)
                    {
                        blankline = 0;
                    }
                    else
                    {
                        blankline = 1;
                    }
                }
                if (0 == blankline && tok.level == 0)
                {
                    if (col == tok.indstack[tok.indent])
                    {
                        if (altcol != tok.altindstack[tok.indent])
                        {
                            if (0 != indenterror(tok))
                            {
                                return(ERRORTOKEN);
                            }
                        }
                    }
                    else if (col > tok.indstack[tok.indent])
                    {
                        if (tok.indent + 1 >= MAXINDENT)
                        {
                            tok.done = E_TOODEEP;
                            tok.cur  = tok.inp;
                            return(ERRORTOKEN);
                        }
                        if (altcol <= tok.altindstack[tok.indent])
                        {
                            if (0 != indenterror(tok))
                            {
                                return(ERRORTOKEN);
                            }
                        }
                        tok.pendin++;
                        tok.indstack[++tok.indent]  = col;
                        tok.altindstack[tok.indent] = altcol;
                    }
                    else
                    {
                        while (tok.indent > 0 &&
                               col < tok.indstack[tok.indent])
                        {
                            tok.pendin--;
                            tok.indent--;
                        }
                        if (col != tok.indstack[tok.indent])
                        {
                            tok.done = E_DEDENT;
                            tok.cur  = tok.inp;
                            return(ERRORTOKEN);
                        }
                        if (altcol != tok.altindstack[tok.indent])
                        {
                            if (0 != indenterror(tok))
                            {
                                return(ERRORTOKEN);
                            }
                        }
                    }
                }
            }

            tok.start = tok.cur;

            if (tok.pendin != 0)
            {
                if (tok.pendin < 0)
                {
                    tok.pendin++;
                    return(DEDENT);
                }
                else
                {
                    tok.pendin--;
                    return(INDENT);
                }
            }

again:
            tok.start = null;
            do
            {
                c = tok_nextc(tok);
            } while (c == ' ' || c == '\t' || c == '\x0C');             //c == '\014');

            tok.start = tok.cur - 1;

            if (c == '#')
            {
                CharPtr cbuf = new CharPtr(new char[80]);
                CharPtr tp; int cp;
                tp = new CharPtr(cbuf);
                do
                {
                    c = tok_nextc(tok); tp[0] = (char)c; tp.inc();
                } while (c != EOF && c != '\n' &&
                         tp - cbuf + 1 < 80 /*sizeof(cbuf)*/);
                tp[0] = '\0';
                for (cp = 0;
                     cp < PyTokenizer_Get_tabforms.Length;
                     cp++)
                {
                    if (null != (tp = strstr(cbuf, PyTokenizer_Get_tabforms[cp])))
                    {
                        int newsize = atoi(tp + strlen(PyTokenizer_Get_tabforms[cp]));

                        if (newsize >= 1 && newsize <= 40)
                        {
                            tok.tabsize = newsize;
                            if (0 != Py_VerboseFlag)
                            {
                                PySys_WriteStderr(
                                    "Tab size set to %d\n",
                                    newsize);
                            }
                        }
                    }
                }
                while (c != EOF && c != '\n')
                {
                    c = tok_nextc(tok);
                }
            }

            if (c == EOF)
            {
                return(tok.done == E_EOF ? ENDMARKER : ERRORTOKEN);
            }


            if (isalpha(c) || c == '_')
            {
                switch (c)
                {
                case 'r':
                case 'R':
                    c = tok_nextc(tok);
                    if (c == '"' || c == '\'')
                    {
                        goto letter_quote;
                    }
                    break;

                case 'u':
                case 'U':
                    c = tok_nextc(tok);
                    if (c == 'r' || c == 'R')
                    {
                        c = tok_nextc(tok);
                    }
                    if (c == '"' || c == '\'')
                    {
                        goto letter_quote;
                    }
                    break;
                }
                while (isalnum(c) || c == '_')
                {
                    c = tok_nextc(tok);
                }
                tok_backup(tok, c);
                p_start = new CharPtr(tok.start);
                p_end   = new CharPtr(tok.cur);
                return(NAME);
            }

            if (c == '\n')
            {
                tok.atbol = 1;
                if (0 != blankline || tok.level > 0)
                {
                    goto nextline;
                }
                p_start = new CharPtr(tok.start);
                p_end   = new CharPtr(tok.cur - 1);
                return(NEWLINE);
            }

            if (c == '.')
            {
                c = tok_nextc(tok);
                if (isdigit(c))
                {
                    do
                    {
                        c = tok_nextc(tok);
                    } while (isdigit(c));
                    if (c == 'e' || c == 'E')
                    {
                        c = tok_nextc(tok);
                        if (c == '+' || c == '-')
                        {
                            c = tok_nextc(tok);
                        }
                        if (!isdigit(c))
                        {
                            tok.done = E_TOKEN;
                            tok_backup(tok, c);
                            return(ERRORTOKEN);
                        }
                        do
                        {
                            c = tok_nextc(tok);
                        } while (isdigit(c));
                    }
                    if (c == 'j' || c == 'J')
                    {
                        c = tok_nextc(tok);
                    }
                    //goto fraction;
                    tok_backup(tok, c);
                    p_start = new CharPtr(tok.start);
                    p_end   = new CharPtr(tok.cur);
                    return(NUMBER);
                }
                else
                {
                    tok_backup(tok, c);
                    p_start = new CharPtr(tok.start);
                    p_end   = new CharPtr(tok.cur);
                    return(DOT);
                }
            }

            if (isdigit(c))
            {
                if (c == '0')
                {
                    c = tok_nextc(tok);
                    if (c == '.')
                    {
                        do
                        {
                            c = tok_nextc(tok);
                        } while (isdigit(c));
                        if (c == 'e' || c == 'E')
                        {
                            c = tok_nextc(tok);
                            if (c == '+' || c == '-')
                            {
                                c = tok_nextc(tok);
                            }
                            if (!isdigit(c))
                            {
                                tok.done = E_TOKEN;
                                tok_backup(tok, c);
                                return(ERRORTOKEN);
                            }
                            do
                            {
                                c = tok_nextc(tok);
                            } while (isdigit(c));
                        }
                        if (c == 'j' || c == 'J')
                        {
                            c = tok_nextc(tok);
                        }
                        goto fraction;
                    }
                    if (c == 'j' || c == 'J')
                    {
                        c = tok_nextc(tok);
                        goto imaginary;
                    }
                    if (c == 'x' || c == 'X')
                    {
                        do
                        {
                            c = tok_nextc(tok);
                        } while (isxdigit(c));
                    }
                    else
                    {
                        int found_decimal = 0;
                        while ('0' <= c && c < '8')
                        {
                            c = tok_nextc(tok);
                        }
                        if (isdigit(c))
                        {
                            found_decimal = 1;
                            do
                            {
                                c = tok_nextc(tok);
                            } while (isdigit(c));
                        }
                        if (c == '.')
                        {
                            do
                            {
                                c = tok_nextc(tok);
                            } while (isdigit(c));
                            if (c == 'e' || c == 'E')
                            {
                                c = tok_nextc(tok);
                                if (c == '+' || c == '-')
                                {
                                    c = tok_nextc(tok);
                                }
                                if (!isdigit(c))
                                {
                                    tok.done = E_TOKEN;
                                    tok_backup(tok, c);
                                    return(ERRORTOKEN);
                                }
                                do
                                {
                                    c = tok_nextc(tok);
                                } while (isdigit(c));
                            }
                            if (c == 'j' || c == 'J')
                            {
                                c = tok_nextc(tok);
                            }
                            goto fraction;
                        }
                        else if (c == 'e' || c == 'E')
                        {
                            c = tok_nextc(tok);
                            if (c == '+' || c == '-')
                            {
                                c = tok_nextc(tok);
                            }
                            if (!isdigit(c))
                            {
                                tok.done = E_TOKEN;
                                tok_backup(tok, c);
                                return(ERRORTOKEN);
                            }
                            do
                            {
                                c = tok_nextc(tok);
                            } while (isdigit(c));
                            if (c == 'j' || c == 'J')
                            {
                                c = tok_nextc(tok);
                            }
                            goto exponent;
                        }

                        else if (c == 'j' || c == 'J')
                        {
                            c = tok_nextc(tok);
                            goto imaginary;
                        }

                        else if (0 != found_decimal)
                        {
                            tok.done = E_TOKEN;
                            tok_backup(tok, c);
                            return(ERRORTOKEN);
                        }
                    }
                    if (c == 'l' || c == 'L')
                    {
                        c = tok_nextc(tok);
                    }
                }
                else
                {
                    do
                    {
                        c = tok_nextc(tok);
                    } while (isdigit(c));
                    if (c == 'l' || c == 'L')
                    {
                        c = tok_nextc(tok);
                    }
                    else
                    {
                        if (c == '.')
                        {
                            //fraction:
                            do
                            {
                                c = tok_nextc(tok);
                            } while (isdigit(c));
                        }
                        if (c == 'e' || c == 'E')
                        {
                            //exponent:
                            c = tok_nextc(tok);
                            if (c == '+' || c == '-')
                            {
                                c = tok_nextc(tok);
                            }
                            if (!isdigit(c))
                            {
                                tok.done = E_TOKEN;
                                tok_backup(tok, c);
                                return(ERRORTOKEN);
                            }
                            do
                            {
                                c = tok_nextc(tok);
                            } while (isdigit(c));
                        }
                        if (c == 'j' || c == 'J')
                        {
                            //imaginary:
                            c = tok_nextc(tok);
                        }
                    }
                }
fraction:
exponent:
imaginary:
                tok_backup(tok, c);
                p_start = new CharPtr(tok.start);
                p_end   = new CharPtr(tok.cur);
                return(NUMBER);
            }

letter_quote:
            if (c == '\'' || c == '"')
            {
                int quote2    = tok.cur - tok.start + 1;
                int quote     = c;
                int triple    = 0;
                int tripcount = 0;
                for (;;)
                {
                    c = tok_nextc(tok);
                    if (c == '\n')
                    {
                        if (0 == triple)
                        {
                            tok.done = E_TOKEN;
                            tok_backup(tok, c);
                            return(ERRORTOKEN);
                        }
                        tripcount = 0;
                    }
                    else if (c == EOF)
                    {
                        tok.done = E_TOKEN;
                        tok.cur  = tok.inp;
                        return(ERRORTOKEN);
                    }
                    else if (c == quote)
                    {
                        tripcount++;
                        if (tok.cur - tok.start == quote2)
                        {
                            c = tok_nextc(tok);
                            if (c == quote)
                            {
                                triple    = 1;
                                tripcount = 0;
                                continue;
                            }
                            tok_backup(tok, c);
                        }
                        if (0 == triple || tripcount == 3)
                        {
                            break;
                        }
                    }
                    else if (c == '\\')
                    {
                        tripcount = 0;
                        c         = tok_nextc(tok);
                        if (c == EOF)
                        {
                            tok.done = E_TOKEN;
                            tok.cur  = tok.inp;
                            return(ERRORTOKEN);
                        }
                    }
                    else
                    {
                        tripcount = 0;
                    }
                }
                p_start = new CharPtr(tok.start);
                p_end   = new CharPtr(tok.cur);
                return(STRING);
            }

            if (c == '\\')
            {
                c = tok_nextc(tok);
                if (c != '\n')
                {
                    tok.done = E_TOKEN;
                    tok.cur  = tok.inp;
                    return(ERRORTOKEN);
                }
                goto again;
            }

            {
                int c2    = tok_nextc(tok);
                int token = PyToken_TwoChars(c, c2);
                if (token != OP)
                {
                    int c3     = tok_nextc(tok);
                    int token3 = PyToken_ThreeChars(c, c2, c3);
                    if (token3 != OP)
                    {
                        token = token3;
                    }
                    else
                    {
                        tok_backup(tok, c3);
                    }
                    p_start = new CharPtr(tok.start);
                    p_end   = new CharPtr(tok.cur);
                    return(token);
                }
                tok_backup(tok, c2);
            }

            switch (c)
            {
            case '(':
            case '[':
            case '{':
                tok.level++;
                break;

            case ')':
            case ']':
            case '}':
                tok.level--;
                break;
            }

            p_start = new CharPtr(tok.start);
            p_end   = new CharPtr(tok.cur);
            return(PyToken_OneChar(c));
        }
Ejemplo n.º 10
0
        private static int tok_nextc(tok_state tok)
        {
            for (;;)
            {
                if (tok.cur != tok.inp)
                {
                    return(Py_CHARMASK(tok.cur[0])); tok.cur.inc();
                }
                if (tok.done != E_OK)
                {
                    return(EOF);
                }
                if (tok.fp == null)
                {
                    CharPtr end = strchr(tok.inp, '\n');
                    if (end != null)
                    {
                        end.inc();
                    }
                    else
                    {
                        end = strchr(tok.inp, '\0');
                        if (end == tok.inp)
                        {
                            tok.done = E_EOF;
                            return(EOF);
                        }
                    }
                    if (tok.start == null)
                    {
                        tok.buf = tok.cur;
                    }
                    tok.lineno++;
                    tok.inp = end;
                    return(Py_CHARMASK(tok.cur[0])); tok.cur.inc();
                }
                if (tok.prompt != null)
                {
                    CharPtr new_ = PyOS_Readline(tok.prompt);
                    if (tok.nextprompt != null)
                    {
                        tok.prompt = tok.nextprompt;
                    }
                    if (new_ == null)
                    {
                        tok.done = E_INTR;
                    }
                    else if (new_[0] == '\0')
                    {
                        PyMem_FREE(ref new_);
                        tok.done = E_EOF;
                    }
                    else if (tok.start != null)
                    {
                        size_t  start  = (size_t)(tok.start - tok.buf);
                        size_t  oldlen = (size_t)(tok.cur - tok.buf);
                        size_t  newlen = oldlen + strlen(new_);
                        CharPtr buf    = new CharPtr(tok.buf);
                        PyMem_RESIZE_char(ref buf, (int)(newlen + 1));
                        tok.lineno++;
                        if (buf == null)
                        {
                            PyMem_DEL(tok.buf);
                            tok.buf = null;
                            PyMem_FREE(ref new_);
                            tok.done = E_NOMEM;
                            return(EOF);
                        }
                        tok.buf = buf;
                        tok.cur = tok.buf + oldlen;
                        strcpy(tok.buf + oldlen, new_);
                        PyMem_FREE(ref new_);
                        tok.inp   = tok.buf + newlen;
                        tok.end   = tok.inp + 1;
                        tok.start = tok.buf + start;
                    }
                    else
                    {
                        tok.lineno++;
                        if (tok.buf != null)
                        {
                            PyMem_DEL(tok.buf);
                        }
                        tok.buf = new CharPtr(new_);
                        tok.cur = new CharPtr(tok.buf);
                        tok.inp = strchr(tok.buf, '\0');
                        tok.end = tok.inp + 1;
                    }
                }
                else
                {
                    int     done = 0;
                    int     cur  = 0;
                    CharPtr pt;
                    if (tok.start == null)
                    {
                        if (tok.buf == null)
                        {
                            tok.buf = PyMem_NEW_char2(BUFSIZ);
                            if (tok.buf == null)
                            {
                                tok.done = E_NOMEM;
                                return(EOF);
                            }
                            tok.end = tok.buf + BUFSIZ;
                        }
                        if (fgets(tok.buf, (int)(tok.end - tok.buf),
                                  tok.fp) == null)
                        {
                            tok.done = E_EOF;
                            done     = 1;
                        }
                        else
                        {
                            tok.done = E_OK;
                            tok.inp  = strchr(tok.buf, '\0');
                            done     = ((tok.inp[-1] == '\n')?1:0);
                        }
                    }
                    else
                    {
                        cur = tok.cur - tok.buf;
                        if (0 != feof(tok.fp))
                        {
                            tok.done = E_EOF;
                            done     = 1;
                        }
                        else
                        {
                            tok.done = E_OK;
                        }
                    }
                    tok.lineno++;
                    while (0 == done)
                    {
                        int curstart = tok.start == null ? -1 :
                                       tok.start - tok.buf;
                        int     curvalid = tok.inp - tok.buf;
                        int     newsize  = curvalid + BUFSIZ;
                        CharPtr newbuf   = new CharPtr(tok.buf);
                        PyMem_RESIZE_char(ref newbuf, newsize);
                        if (newbuf == null)
                        {
                            tok.done = E_NOMEM;
                            tok.cur  = tok.inp;
                            return(EOF);
                        }
                        tok.buf   = newbuf;
                        tok.inp   = tok.buf + curvalid;
                        tok.end   = tok.buf + newsize;
                        tok.start = curstart < 0 ? null :
                                    tok.buf + curstart;
                        if (fgets(tok.inp,
                                  (int)(tok.end - tok.inp),
                                  tok.fp) == null)
                        {
                            strcpy(tok.inp, "\n");
                        }
                        tok.inp = strchr(tok.inp, '\0');
                        done    = ((tok.inp[-1] == '\n')?1:0);
                    }
                    tok.cur = tok.buf + cur;

                    pt = tok.inp - 2;
                    if (pt >= tok.buf && pt[0] == '\r')
                    {
                        pt[0]   = '\n'; pt.inc();
                        pt[0]   = '\0';
                        tok.inp = new CharPtr(pt);
                    }
                }
                if (tok.done != E_OK)
                {
                    if (tok.prompt != null)
                    {
                        PySys_WriteStderr("\n");
                    }
                    tok.cur = tok.inp;
                    return(EOF);
                }
            }
        }