Beispiel #1
0
        void error(string msg, int line = -1, int col = -1, Token tok = null)
        {
            if (tok == null)
                tok = reader.Peek();

            if (line == -1)
                line = tok.Line;
            if (col == -1)
                col = tok.Column;
            msg += ", got '" + tok.Data + "'";

            LuaSourceException ex = new LuaSourceException(line, col, msg);
            Errors.Add(ex);
            //Console.WriteLine(ex.GenerateMessage("sd"));
            if (ThrowParsingErrors)
                throw ex;
        }
Beispiel #2
0
        public TokenReader Lex(string s)
        {
            List<Token> tokens = new List<Token>();
            src = s;
            p = 0;
            ln = 1;
            col = 1;

            while (true)
            {
                List<Token> leading = new List<Token>();
                // eat whitespace/comments
                while (true)
                {
                    char c_ = peek();
                    if (c_ == '#' && peek(1) == '!' && ln == 1)
                    {
                        // linux shebang
                        string sh = "";
                        while (peek() != '\n' && peek() != '\r' && peek() != '\0')
                        {
                            sh += read();
                        }
                        //readnl();
                        leading.Add(new Token { Type = TokenType.Shebang, Data = sh });
                    }
                    else if (c_ == ' ' || c_ == '\t')
                    {
                        leading.Add(new Token { Type = c_ == ' ' ? TokenType.WhitespaceSpace : TokenType.WhitespaceTab, Data = c_.ToString() });
                        read();
                    }
                    else if (c_ == '-' && peek(1) == '-')
                    {
                        read();
                        read();
                        char openParen = '\0';
                        if (peek() == '[')
                            openParen = read();
                        string comment = tryReadLongStr();
                        if (comment == null)
                        {
                            comment = "--" + (openParen != '\0' ? openParen.ToString() : "");
                            while (peek() != '\n' && peek() != '\r' && peek() != '\0')
                                comment += read();
                            //read();
                            //if (peek() == '\n') // e.g. \r\n
                            //    read();
                            if (comment.Length >= 3 && comment.Substring(0, 3) == "---")
                                leading.Add(new Token { Type = TokenType.DocumentationComment, Data = comment });
                            else
                                leading.Add(new Token { Type = TokenType.ShortComment, Data = comment });
                        }
                        else
                        {
                            comment = "--" + comment;
                            leading.Add(new Token { Type = TokenType.LongComment, Data = comment });
                        }
                    }
                    else if (c_ == '\n' || c_ == '\r')
                    {
                        leading.Add(new Token { Type = c_ == '\n' ? TokenType.WhitespaceN : TokenType.WhitespaceR, Data = c_.ToString() });
                        // read handles line changing...
                        read();
                    }
                    else
                        break;
                }

                Token t = new Token();
                t.Leading = leading;
                t.Line = ln;
                t.Column = col;

                char c = read();

                if (c == '\0')
                    t.Type = TokenType.EndOfStream;
                else if (char.IsLetter(c) || c == '_')
                {
                    // ident / keyword
                    string s4 = c.ToString();
                    while (char.IsLetter(peek()) ||
                        peek() == '_' ||
                        char.IsDigit(peek()) &&
                        peek() != '\0')
                    {
                        s4 += read();
                    }
                    t.Data = s4;
                    if (IsKeyword(s4))
                        t.Type = TokenType.Keyword;
                    else
                        t.Type = TokenType.Ident;
                }
                else if (char.IsDigit(c) ||
                    (c == '.' && char.IsDigit(peek())))
                { // negative numbers are handled in unary minus collection
                    string num = "";
                    if (c == '0' && matchpeek("xX"))
                    {
                        //read(); -> already done
                        num = "0" + read(); // 'xX'
                        while (IsHexDigit(peek()))
                            num += read();
                    }
#if !VANILLA_LUA
                    else if (c == '0' && matchpeek("bB"))
                    {
                        num = "0" + read(); // read 'bB'
                        while (char.IsDigit(peek()) || peek() == '_')
                            num += read();
                    }
                    else if (c == '0' && matchpeek("oO"))
                    {
                        num = "0" + read(); // read 'oO'
                        while (char.IsDigit(peek()) || peek() == '_')
                            num += read();
                    }
#endif
                    else
                    {
                        num = c.ToString();
                        bool dec = false;
                        while (char.IsDigit(peek())
                            || peek() == '.'
#if !VANILLA_LUA
 || peek() == '_')
#else
                            )
#endif
                        {
                            num += read();
                            if (peek() == '.')
                            {
                                if (dec)
                                    error("Number has more than one decimal point");
                                dec = true;
                                num += read();
                            }
                        }
                    }
#if !VANILLA_LUA
                    if (matchpeek("PpEe")) // exponent
                    {
                        num += read();
                        if (matchpeek("+-"))
                            num += read();
                        while (char.IsDigit(peek()))
                            num += read();
                    }
#endif
                    t.Data = num;
                    t.Type = TokenType.Number;
                }
                else if (c == '\'' || c == '"')
                {
                    char delim = c;
                    string str = "";
                    while (true)
                    {
                        char c2 = read();
                        if (c2 == '\\')
                        {
                            str += "\\";
                            str += read(); // we won't parse \0xFF, \000, \n, etc here
                        }
                        else if (c2 == delim)
                            break;
                        else if (c2 == '\0')
                            error("expected '" + delim + "', not <eof>");
                        else
                            str += c2;
                    }
                    t.Data = str;
                    t.Type = delim == '"' ? TokenType.DoubleQuoteString : TokenType.SingleQuoteString;
                }
                else if (c == '[')
                {
                    string s3 = tryReadLongStr();
                    if (s3 == null)
                    {
                        t.Type = TokenType.Symbol;
                        t.Data = "[";
                    }
                    else
                    {
                        t.Type = TokenType.LongString;
                        t.Data = s3;
                    }
                }
                else if (c == '<' || c == '>' || c == '=')
                {
                    t.Type = TokenType.Symbol;
                    if (peek() == '=' ||
                        (c == '<' && peek() == '<') ||
                        (c == '>' && peek() == '>'))
                    {
                        t.Data = c.ToString() + read().ToString();
#if !VANILLA_LUA
                        if (peek() == '=' && (c == '<' || c == '>'))
                            t.Data += read(); // augmented, e.g. >>=, but not ===
#endif
                    }
                    else
                        t.Data = c.ToString();
                }
                else if (c == '~')
                {
                    if (peek() == '=')
                    {
                        read();
                        t.Type = TokenType.Symbol;
                        t.Data = "~=";
                    }
                    else
                    {
                        t.Type = TokenType.Symbol;
                        t.Data = "~";
                    }

                }
                else if (c == '.')
                {
                    t.Type = TokenType.Symbol;
                    if (peek() == '.')
                    {
                        read(); // read second '.
                        if (peek() == '.')
                        {
                            t.Data = "...";
                            read(); // read third '.'
                        }
#if !VANILLA_LUA
                        else if (peek() == '=') // ..=
                        {
                            t.Data = "..=";
                            read(); // read '='
                        }
#endif
                        else
                            t.Data = "..";
                    }
                    else
                    {
                        t.Data = ".";
                    }
                }
                else if (c == ':')
                {
                    t.Type = TokenType.Symbol;
#if !VANILLA_LUA
                    if (peek() == ':')
                    {
                        read();
                        t.Data = "::";
                    }
                    else
#endif
                        t.Data = ":";
                }
                else if (c == '-' && peek() == '>')
                {
                    read(); // read the '>'
                    t.Data = "->";
                    t.Type = TokenType.Symbol;
                }
                else if (c == '^')
                {
                    t.Type = TokenType.Symbol;

                    if (peek() == '^')
                    {
                        read();
#if !VANILLA_LUA
                        if (peek() == '=')
                        {
                            read();
                            t.Data = "^^=";
                        }
                        else
#endif
                            t.Data = "^^";
                    }
#if !VANILLA_LUA
                    else if (peek() == '=')
                    {
                        read();
                        t.Data = "^=";
                    }
#endif
                    else
                        t.Data = "^";
                }
#if !VANILLA_LUA
                else if (c == '!')
                {
                    t.Type = TokenType.Symbol;
                    if (peek() == '=')
                    {
                        read();
                        t.Data = "!=";
                    }
                    else t.Data = "!";
                }
#endif
                else if (IsSymbol(c))
                {
                    t.Type = TokenType.Symbol;
                    t.Data = c.ToString();
#if !VANILLA_LUA
                    if (peek() == '=')
                    {
                        char c2 = peek();
                        if (c == '+' ||
                            c == '-' ||
                            c == '/' ||
                            c == '*' ||
                            c == '^' ||
                            c == '%' ||
                            c == '&' ||
                            c == '|')
                        {
                            t.Data += "=";
                            read();
                        }
                    }
#endif
                }
                else
                {
                    p--; // un-read token
                    col--;
                    error("Unexpected Symbol '" + c + "'");
                    read();
                }

                tokens.Add(t);

                if (peek() == '\0')
                    break;
            }
            if (tokens.Count > 0 && tokens[tokens.Count - 1].Type != TokenType.EndOfStream)
                tokens.Add(new Token { Type = TokenType.EndOfStream });
            if (tokens.Count > 1) // 2+
                tokens[tokens.Count - 2].FollowingEoSToken = tokens[tokens.Count - 1];
            return new TokenReader(tokens);
        }
Beispiel #3
0
 public Token(Token copy)
 {
     this.token = copy.token;
     this.seminfo = new SemInfo(copy.seminfo);
 }
Beispiel #4
0
        bool isAugmentedAssignment(Token token)
        {
            switch (token.Data)
            {
                case "..=":
                case ">>=":
                case "<<=":
                case "+=":
                case "-=":
                case "/=":
                case "*=":
                case "&=":
                case "|=":
                case "^=":
                case "%=":
                case "^^=":
                    return true;

                default:
                    return false;
            }
        }