Ejemplo n.º 1
0
        public void GetNextToken()
        {
            PreviousTokenType = TokenType;
            TokenType         = LexTypes.Eof;
            TokenString       = string.Empty;

            while (CurrentChar != (char)0 && CurrentChar.IsWhitespace())
            {
                GetNextChar();
            }

            //single line comment
            if (CurrentChar == '/' && NextChar == '/')
            {
                while (CurrentChar != 0 && CurrentChar != '\n')
                {
                    GetNextChar();
                }
                GetNextChar();
                GetNextToken();
                return;
            }

            //multi line comment
            if (CurrentChar == '/' && NextChar == '*')
            {
                while (CurrentChar != 0 && (CurrentChar != '*' || NextChar != '/'))
                {
                    GetNextChar();
                }
                GetNextChar();
                GetNextChar();
                GetNextToken();
                return;
            }

            TokenStart = dataPos - 2;

            if (CurrentChar.IsAlpha()) //IDs
            {
                while (CurrentChar.IsAlpha() || CurrentChar.IsNumeric())
                {
                    TokenString += CurrentChar;
                    GetNextChar();
                }

                TokenType = LexTypes.Id;
                switch (TokenString)
                {
                case "if": TokenType = LexTypes.RIf; break;

                case "else": TokenType = LexTypes.RElse; break;

                case "do": TokenType = LexTypes.RDo; break;

                case "while": TokenType = LexTypes.RWhile; break;

                case "for": TokenType = LexTypes.RFor; break;

                case "break": TokenType = LexTypes.RBreak; break;

                case "continue": TokenType = LexTypes.RContinue; break;

                case "function": TokenType = LexTypes.RFunction; break;

                case "return": TokenType = LexTypes.RReturn; break;

                case "var": TokenType = LexTypes.RVar; break;

                case "true": TokenType = LexTypes.RTrue; break;

                case "false": TokenType = LexTypes.RFalse; break;

                case "null": TokenType = LexTypes.RNull; break;

                case "undefined": TokenType = LexTypes.RUndefined; break;

                case "new": TokenType = LexTypes.RNew; break;

                case "typeof": TokenType = LexTypes.RTypeOf; break;

                case "try": TokenType = LexTypes.RTry; break;

                case "catch": TokenType = LexTypes.RCatch; break;

                case "finally": TokenType = LexTypes.RFinally; break;

                case "throw": TokenType = LexTypes.RThrow; break;

                case "const": TokenType = LexTypes.RConst; break;
                }
            }
            else if (CurrentChar.IsNumeric()) //Numbers
            {
                var isHex = false;
                if (CurrentChar == '0')
                {
                    TokenString += CurrentChar;
                    GetNextChar();
                }

                if (CurrentChar == 'x')
                {
                    isHex        = true;
                    TokenString += CurrentChar;
                    GetNextChar();
                }

                TokenType = LexTypes.Int;

                while (CurrentChar.IsNumeric() || (isHex && CurrentChar.IsHexadecimal()))
                {
                    TokenString += CurrentChar;
                    GetNextChar();
                }

                if (!isHex && CurrentChar == '.')
                {
                    TokenType    = LexTypes.Float;
                    TokenString += '.';
                    GetNextChar();
                    while (CurrentChar.IsNumeric())
                    {
                        TokenString += CurrentChar;
                        GetNextChar();
                    }
                }

                if (!isHex && (CurrentChar == 'e' || CurrentChar == 'E'))
                {
                    TokenType    = LexTypes.Float;
                    TokenString += CurrentChar;
                    GetNextChar();
                    if (CurrentChar == '-')
                    {
                        TokenString += CurrentChar;
                        GetNextChar();
                    }
                    while (CurrentChar.IsNumeric())
                    {
                        TokenString += CurrentChar;
                        GetNextChar();
                    }
                }
            }
            else if (CurrentChar == '\'' || CurrentChar == '\"') //Strings again
            {
                var endChar = CurrentChar;
                GetNextChar();

                while (CurrentChar != (char)0 && CurrentChar != endChar)
                {
                    if (CurrentChar == '\\')
                    {
                        GetNextChar();

                        switch (CurrentChar)
                        {
                        case '\n': break;

                        case 'n':
                            TokenString += '\n';
                            break;

                        case 'r':
                            TokenString += '\r';
                            break;

                        case 'a':
                            TokenString += '\a';
                            break;

                        case 'b':
                            TokenString += '\b';
                            break;

                        case 'f':
                            TokenString += '\f';
                            break;

                        case 't':
                            TokenString += '\t';
                            break;

                        case 'v':
                            TokenString += '\v';
                            break;

                        case 'x':
                        {
                            var str = "";
                            GetNextChar();
                            str += CurrentChar;
                            GetNextChar();
                            str         += CurrentChar;
                            TokenString += (char)Convert.ToInt64(str);
                        }
                        break;

                        default:
                            if (CurrentChar >= '0' && CurrentChar <= '7')
                            {
                                var str = "";
                                str += CurrentChar;
                                GetNextChar();
                                str += CurrentChar;
                                GetNextChar();
                                str         += CurrentChar;
                                TokenString += (char)Convert.ToInt64(str);
                            }
                            else
                            {
                                TokenString += CurrentChar;
                            }
                            break;
                        }
                    }
                    else
                    {
                        TokenString += CurrentChar;
                    }

                    GetNextChar();
                }

                GetNextChar();

                TokenType = LexTypes.Str;
            }
            else //Single character
            {
                TokenType = (LexTypes)CurrentChar;

                if (CurrentChar != (char)0)
                {
                    GetNextChar();
                }

                if (TokenType == (LexTypes)'=' && CurrentChar == '=') // ==
                {
                    TokenType = LexTypes.Equal;
                    GetNextChar();

                    if (CurrentChar == '=') //===
                    {
                        TokenType = LexTypes.TypeEqual;
                        GetNextChar();
                    }
                }
                else if (TokenType == (LexTypes)'!' && CurrentChar == '=') // !=
                {
                    TokenType = LexTypes.NEqual;
                    GetNextChar();
                    if (CurrentChar == '=') // !==
                    {
                        TokenType = LexTypes.NTypeEqual;
                        GetNextChar();
                    }
                }
                else if (TokenType == (LexTypes)'<' && CurrentChar == '=') // <=
                {
                    TokenType = LexTypes.LEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'<' && CurrentChar == '<') // <<
                {
                    TokenType = LexTypes.LShift;
                    GetNextChar();
                    if (CurrentChar == '=') //<<=
                    {
                        TokenType = LexTypes.LShiftEqual;
                        GetNextChar();
                    }
                }
                else if (TokenType == (LexTypes)'>' && CurrentChar == '=') // >=
                {
                    TokenType = LexTypes.GEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'>' && CurrentChar == '>') // >>
                {
                    TokenType = LexTypes.RShift;
                    GetNextChar();

                    if (CurrentChar == '=') // >>=
                    {
                        TokenType = LexTypes.RShiftEqual;
                        GetNextChar();
                    }
                    else if (CurrentChar == '>') // >>>
                    {
                        TokenType = LexTypes.RShiftUnsigned;
                        GetNextChar();

                        if (CurrentChar == '=') // >>>=
                        {
                            TokenType = LexTypes.RShiftUnsignedEqual;
                            GetNextChar();
                        }
                    }
                }
                else if (TokenType == (LexTypes)'+' && CurrentChar == '=') // +=
                {
                    TokenType = LexTypes.PlusEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'-' && CurrentChar == '=') // -=
                {
                    TokenType = LexTypes.MinusEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'+' && CurrentChar == '+') // ++
                {
                    TokenType = LexTypes.PlusPlus;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'-' && CurrentChar == '-') // --
                {
                    TokenType = LexTypes.MinusMinus;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'&' && CurrentChar == '=') // &=
                {
                    TokenType = LexTypes.AndEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'&' && CurrentChar == '&') // &&
                {
                    TokenType = LexTypes.AndAnd;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'|' && CurrentChar == '=') // |=
                {
                    TokenType = LexTypes.OrEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'|' && CurrentChar == '|') // ||
                {
                    TokenType = LexTypes.OrOr;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'^' && CurrentChar == '=') // ^=
                {
                    TokenType = LexTypes.XorEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'/')
                {
                    //omit regex for now

                    TokenType = LexTypes.RegExp;
                    foreach (var item in notAllowedBeforeRegex)
                    {
                        if (item == PreviousTokenType)
                        {
                            TokenType = (LexTypes)'/';
                            break;
                        }
                    }

                    if (TokenType == LexTypes.RegExp)
                    {
                        TokenString = "/";

                        while (CurrentChar != 0 && CurrentChar != '/' && CurrentChar != '\n')
                        {
                            if (CurrentChar == '\\' && NextChar == '/')
                            {
                                TokenString += CurrentChar;
                                GetNextChar();
                            }

                            TokenString += CurrentChar;
                            GetNextChar();
                        }

                        if (CurrentChar == '/')
                        {
                            var regexStr = TokenString.Substring(1);
                            try
                            {
                                var regexObj = new Regex(regexStr, RegexOptions.ECMAScript);
                            }
                            catch (Exception ex)
                            {
                                throw new ScriptException("Invalid RegEx", ex);
                            }

                            do
                            {
                                TokenString += CurrentChar;
                                GetNextChar();
                            } while (CurrentChar == 'g' || CurrentChar == 'i' || CurrentChar == 'm' || CurrentChar == 'y');
                        }
                        else
                        {
                        }
                    }
                    else if (CurrentChar == '=') // /=
                    {
                        TokenType = LexTypes.SlashEqual;
                        GetNextChar();
                    }
                }
                else if (TokenType == (LexTypes)'%' && CurrentChar == '=') // %=
                {
                    TokenType = LexTypes.PercentEqual;
                    GetNextChar();
                }
            }

            /* Something broke... */
            TokenLastEnd = TokenEnd;
            TokenEnd     = dataPos - 3;
        }
Ejemplo n.º 2
0
        public void GetNextToken()
        {
            ColumnNumber = _currColumnNumber;

            TokenType   = LexTypes.Eof;
            TokenString = String.Empty;

            while (CurrentChar != (char)0 && CurrentChar.IsWhitespace())
            {
                GetNextChar();
            }

            //single line comment
            if (CurrentChar == '/' && NextChar == '/')
            {
                while (CurrentChar != 0 && CurrentChar != '\n')
                {
                    GetNextChar();
                }
                GetNextChar();
                GetNextToken();
                return;
            }

            //multi line comment
            if (CurrentChar == '/' && NextChar == '*')
            {
                while (CurrentChar != '*' && CurrentChar != '/')
                {
                    GetNextChar();
                }
                GetNextChar();
                GetNextChar();
                GetNextToken();
                return;
            }

            TokenStart = _dataPos - 2;

            if (CurrentChar.IsAlpha())             //IDs
            {
                while (CurrentChar.IsAlpha() || CurrentChar.IsNumeric())
                {
                    TokenString += CurrentChar;
                    GetNextChar();
                }

                TokenType = LexTypes.Id;
                switch (TokenString)
                {
                case "if": TokenType = LexTypes.RIf; break;

                case "else": TokenType = LexTypes.RElse; break;

                case "do": TokenType = LexTypes.RDo; break;

                case "while": TokenType = LexTypes.RWhile; break;

                case "for": TokenType = LexTypes.RFor; break;

                case "break": TokenType = LexTypes.RBreak; break;

                case "continue": TokenType = LexTypes.RContinue; break;

                case "function": TokenType = LexTypes.RFunction; break;

                case "return": TokenType = LexTypes.RReturn; break;

                case "var": TokenType = LexTypes.RVar; break;

                case "true": TokenType = LexTypes.RTrue; break;

                case "false": TokenType = LexTypes.RFalse; break;

                case "null": TokenType = LexTypes.RFalse; break;

                case "undefined": TokenType = LexTypes.RNull; break;

                case "new": TokenType = LexTypes.RNew; break;
                }
            }
            else if (CurrentChar.IsNumeric())             //Numbers
            {
                bool isHex = false;
                if (CurrentChar == '0')
                {
                    TokenString += CurrentChar;
                    GetNextChar();
                }

                if (CurrentChar == 'x')
                {
                    isHex        = true;
                    TokenString += CurrentChar;
                    GetNextChar();
                }

                TokenType = LexTypes.Int;

                while (CurrentChar.IsNumeric() || (isHex && CurrentChar.IsHexadecimal()))
                {
                    TokenString += CurrentChar;
                    GetNextChar();
                }

                if (!isHex && CurrentChar == '.')
                {
                    TokenType    = LexTypes.Float;
                    TokenString += '.';
                    GetNextChar();
                    while (CurrentChar.IsNumeric())
                    {
                        TokenString += CurrentChar;
                        GetNextChar();
                    }
                }

                if (!isHex && (CurrentChar == 'e' || CurrentChar == 'E'))
                {
                    TokenType    = LexTypes.Float;
                    TokenString += CurrentChar;
                    GetNextChar();
                    if (CurrentChar == '-')
                    {
                        TokenString += CurrentChar;
                        GetNextChar();
                    }
                    while (CurrentChar.IsNumeric())
                    {
                        TokenString += CurrentChar;
                        GetNextChar();
                    }
                }
            }
            else if (CurrentChar == '"')             //Strings
            {
                GetNextChar();
                while (CurrentChar != (char)0 && CurrentChar != '"')
                {
                    if (CurrentChar == '\\')
                    {
                        GetNextChar();
                        switch (CurrentChar)
                        {
                        case 'n':
                            TokenString += '\n';
                            break;

                        case '"':
                            TokenString += '"';
                            break;

                        case '\\':
                            TokenString += '\\';
                            break;

                        default:
                            TokenString += CurrentChar;
                            break;
                        }
                    }
                    else
                    {
                        TokenString += CurrentChar;
                    }

                    GetNextChar();
                }

                GetNextChar();

                TokenType = LexTypes.Str;
            }
            else if (CurrentChar == '\'')             //Strings again
            {
                GetNextChar();

                while (CurrentChar != (char)0 && CurrentChar != '\'')
                {
                    if (CurrentChar == '\\')
                    {
                        GetNextChar();

                        switch (CurrentChar)
                        {
                        case 'n':
                            TokenString += '\n';
                            break;

                        case 'a':
                            TokenString += '\a';
                            break;

                        case 'r':
                            TokenString += '\r';
                            break;

                        case 't':
                            TokenString += '\t';
                            break;

                        case '\'':
                            TokenString += '\'';
                            break;

                        case '\\':
                            TokenString += '\\';
                            break;

                        case 'x':
                        {
                            String str = "";
                            GetNextChar();
                            str += CurrentChar;
                            GetNextChar();
                            str         += CurrentChar;
                            TokenString += (char)Convert.ToInt64(str);
                        }
                        break;

                        default:
                            if (CurrentChar >= '0' && CurrentChar <= '7')
                            {
                                String str = "";
                                str += CurrentChar;
                                GetNextChar();
                                str += CurrentChar;
                                GetNextChar();
                                str         += CurrentChar;
                                TokenString += (char)Convert.ToInt64(str);
                            }
                            else
                            {
                                TokenString += CurrentChar;
                            }
                            break;
                        }
                    }
                    else
                    {
                        TokenString += CurrentChar;
                    }

                    GetNextChar();
                }

                GetNextChar();

                TokenType = LexTypes.Str;
            }
            else             //Single character
            {
                TokenType = (LexTypes)CurrentChar;

                if (CurrentChar != (char)0)
                {
                    GetNextChar();
                }

                if (TokenType == (LexTypes)'=' && CurrentChar == '=')                 // ==
                {
                    TokenType = LexTypes.Equal;
                    GetNextChar();

                    if (CurrentChar == '=')                     //===
                    {
                        TokenType = LexTypes.TypeEqual;
                        GetNextChar();
                    }
                }
                else if (TokenType == (LexTypes)'!' && CurrentChar == '=')                 // !=
                {
                    TokenType = LexTypes.NEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'<' && CurrentChar == '=')                 // <=
                {
                    TokenType = LexTypes.LEqual;
                    GetNextChar();

                    if (CurrentChar == '=')                     //!==
                    {
                        TokenType = LexTypes.NTypeEqual;
                        GetNextChar();
                    }
                }
                else if (TokenType == (LexTypes)'<' && CurrentChar == '<')                 // <<
                {
                    TokenType = LexTypes.LShift;
                    GetNextChar();
                    if (CurrentChar == '=')                     //<<=
                    {
                        TokenType = LexTypes.LShiftEqual;
                        GetNextChar();
                    }
                }
                else if (TokenType == (LexTypes)'>' && CurrentChar == '=')                 // >=
                {
                    TokenType = LexTypes.GEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'>' && CurrentChar == '>')                 // >>
                {
                    TokenType = LexTypes.RShift;
                    GetNextChar();

                    if (CurrentChar == '=')                     // >>=
                    {
                        TokenType = LexTypes.RShiftEqual;
                        GetNextChar();
                    }
                    else if (CurrentChar == '>')                     // >>>
                    {
                        TokenType = LexTypes.RShiftUnsigned;
                        GetNextChar();
                    }
                }
                else if (TokenType == (LexTypes)'+' && CurrentChar == '=')                 // +=
                {
                    TokenType = LexTypes.PlusEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'-' && CurrentChar == '=')                 // -=
                {
                    TokenType = LexTypes.MinusEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'+' && CurrentChar == '+')                 // ++
                {
                    TokenType = LexTypes.PlusPlus;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'-' && CurrentChar == '-')                 // --
                {
                    TokenType = LexTypes.MinusMinus;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'&' && CurrentChar == '=')                 // &=
                {
                    TokenType = LexTypes.AndEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'&' && CurrentChar == '&')                 // &&
                {
                    TokenType = LexTypes.AndAnd;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'|' && CurrentChar == '=')                 // |=
                {
                    TokenType = LexTypes.OrEqual;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'|' && CurrentChar == '|')                 // ||
                {
                    TokenType = LexTypes.OrOr;
                    GetNextChar();
                }
                else if (TokenType == (LexTypes)'^' && CurrentChar == '=')                 // ^=
                {
                    TokenType = LexTypes.XorEqual;
                    GetNextChar();
                }
            }

            /* Something broke... */
            TokenLastEnd = TokenEnd;
            TokenEnd     = _dataPos - 3;
        }