Пример #1
0
        public new Tokens GetNextToken()
        {
            for (; ;)
            {
                inString = false;
                isCode   = false;

                Tokens token = base.GetNextToken();
                UpdateTokenPosition();

                switch (token)
                {
                    #region Comments

                // ignored tokens:
                case Tokens.T_WHITESPACE: break;

                case Tokens.T_COMMENT: this.commentsSink.OnComment(this, this.tokenPosition); break;

                case Tokens.T_LINE_COMMENT: this.commentsSink.OnLineComment(this, this.tokenPosition); break;

                case Tokens.T_OPEN_TAG: this.commentsSink.OnOpenTag(this, this.tokenPosition); break;

                case Tokens.T_DOC_COMMENT:
                    // remember token value to be used by the next token and skip the current:
                    this.lastDocComment = new PHPDocBlock(base.GetTokenString(), this.tokenPosition);
                    this.commentsSink.OnPhpDocComment(this, this.lastDocComment);
                    break;

                case Tokens.T_PRAGMA_FILE:
                    sourceUnit.AddSourceFileMapping(tokenPosition.FirstLine, base.GetTokenAsFilePragma());
                    break;

                case Tokens.T_PRAGMA_LINE:
                {
                    int?value = base.GetTokenAsLinePragma();

                    if (value.HasValue)
                    {
                        sourceUnit.AddSourceLineMapping(tokenPosition.FirstLine, value.Value);
                    }
                    else
                    {
                        errors.Add(Warnings.InvalidLinePragma, sourceUnit, tokenPosition);
                    }

                    break;
                }

                case Tokens.T_PRAGMA_DEFAULT_FILE:
                    sourceUnit.AddSourceFileMapping(tokenPosition.FirstLine, SourceUnit.DefaultFile);
                    break;

                case Tokens.T_PRAGMA_DEFAULT_LINE:
                    sourceUnit.AddSourceLineMapping(tokenPosition.FirstLine, SourceUnit.DefaultLine);
                    break;

                    #endregion

                    #region String Semantics

                case Tokens.T_VARIABLE:
                    // exclude initial $ from the name:
                    Debug.Assert(GetTokenChar(0) == '$');
                    tokenSemantics.Object = base.GetTokenSubstring(1);
                    goto default;

                case Tokens.T_STRING:
                    if (inString)
                    {
                        StoreEncapsedString();
                    }
                    else
                    {
                        tokenSemantics.Object = base.GetTokenString();
                    }

                    goto default;

                case Tokens.T_ARRAY:
                case Tokens.T_LIST:
                case Tokens.T_ASSERT:
                    tokenSemantics.Object = base.GetTokenString();      // remember the token string, so we can use these tokens as literals later, case sensitively
                    goto default;

                case Tokens.T_STRING_VARNAME:
                case Tokens.T_NUM_STRING:
                case Tokens.T_ENCAPSED_AND_WHITESPACE:
                case Tokens.T_BAD_CHARACTER:
                    StoreEncapsedString();
                    goto default;

                case Tokens.T_INLINE_HTML:
                    tokenSemantics.Object = base.GetTokenString();
                    goto default;


                // \[uU]#{0-6}
                case Tokens.UnicodeCharCode:
                {
                    Debug.Assert(inString);

                    //if (GetTokenChar(1) == 'u')
                    //{
                    //  if (TokenLength != 2 + 4)
                    //    errors.Add(Warnings.InvalidEscapeSequenceLength, sourceFile, tokenPosition.Short, GetTokenString(), 4);
                    //}
                    //else
                    //{
                    //  if (TokenLength != 2 + 6)
                    //    errors.Add(Warnings.InvalidEscapeSequenceLength, sourceFile, tokenPosition.Short, GetTokenString(), 6);
                    //}

                    int code_point = GetTokenAsInteger(2, 16);

                    try
                    {
                        if ((code_point < 0 || code_point > 0x10ffff) || (code_point >= 0xd800 && code_point <= 0xdfff))
                        {
                            errors.Add(Errors.InvalidCodePoint, SourceUnit, tokenPosition, GetTokenString());
                            StoreEncapsedString("?");
                        }
                        else
                        {
                            StoreEncapsedString(StringUtils.Utf32ToString(code_point));
                        }
                    }
                    catch (ArgumentOutOfRangeException)
                    {
                        errors.Add(Errors.InvalidCodePoint, SourceUnit, tokenPosition, GetTokenString());
                        StoreEncapsedString("?");
                    }
                    token = Tokens.T_STRING;
                    goto default;
                }

                // \C{name}
                case Tokens.UnicodeCharName:
                    Debug.Assert(inString);
                    StoreEncapsedString();                             // N/S
                    token = Tokens.T_STRING;
                    goto default;

                // b?"xxx"
                case Tokens.DoubleQuotedString:
                {
                    bool forceBinaryString = GetTokenChar(0) == 'b';

                    tokenSemantics.Object = GetTokenAsDoublyQuotedString(forceBinaryString ? 1 : 0, encoding, forceBinaryString);
                    token = Tokens.T_CONSTANT_ENCAPSED_STRING;
                    goto default;
                }

                // b?'xxx'
                case Tokens.SingleQuotedString:
                {
                    bool forceBinaryString = GetTokenChar(0) == 'b';

                    tokenSemantics.Object = GetTokenAsSinglyQuotedString(forceBinaryString ? 1 : 0, encoding, forceBinaryString);
                    token = Tokens.T_CONSTANT_ENCAPSED_STRING;
                    goto default;
                }

                    #endregion

                    #region Numeric Semantics

                case Tokens.T_CURLY_OPEN:
                    tokenSemantics.Integer = (int)Tokens.T_CURLY_OPEN;
                    goto default;

                case Tokens.T_CHARACTER:
                    tokenSemantics.Integer = (int)GetTokenChar(0);
                    goto default;

                case Tokens.EscapedCharacter:
                    tokenSemantics.Integer = (int)GetTokenAsEscapedCharacter(0);
                    token = Tokens.T_CHARACTER;
                    goto default;

                case Tokens.T_LINE:
                    // TODO:
                    tokenSemantics.Integer = 1;
                    goto default;

                // "\###"
                case Tokens.OctalCharCode:
                    tokenSemantics.Integer = GetTokenAsInteger(1, 10);
                    token = Tokens.T_CHARACTER;
                    goto default;

                // "\x##"
                case Tokens.HexCharCode:
                    tokenSemantics.Integer = GetTokenAsInteger(2, 16);
                    token = Tokens.T_CHARACTER;
                    goto default;

                // {LNUM}
                case Tokens.ParseDecimalNumber:
                {
                    // [0-9]* - value is either in octal or in decimal
                    if (GetTokenChar(0) == '0')
                    {
                        token = GetTokenAsDecimalNumber(1, 8, ref tokenSemantics);
                    }
                    else
                    {
                        token = GetTokenAsDecimalNumber(0, 10, ref tokenSemantics);
                    }

                    if (token == Tokens.T_DNUMBER)
                    {
                        // conversion to double causes data loss
                        errors.Add(Warnings.TooBigIntegerConversion, SourceUnit, tokenPosition, GetTokenString());
                    }
                    goto default;
                }

                // {HNUM}
                case Tokens.ParseHexadecimalNumber:
                {
                    // parse hexadecimal value
                    token = GetTokenAsDecimalNumber(2, 16, ref tokenSemantics);

                    if (token == Tokens.T_DNUMBER)
                    {
                        // conversion to double causes data loss
                        errors.Add(Warnings.TooBigIntegerConversion, SourceUnit, tokenPosition, GetTokenString());
                    }
                    goto default;
                }

                // {BNUM}
                case Tokens.ParseBinaryNumber:
                    // parse binary number value
                    token = GetTokenAsDecimalNumber(2, 2, ref tokenSemantics);

                    if (token == Tokens.T_DNUMBER)
                    {
                        // conversion to double causes data loss
                        errors.Add(Warnings.TooBigIntegerConversion, SourceUnit, tokenPosition, GetTokenString());
                    }
                    goto default;

                // {DNUM}|{EXPONENT_DNUM}
                case Tokens.ParseDouble:
                    tokenSemantics.Double = GetTokenAsDouble(0);
                    token = Tokens.T_DNUMBER;
                    goto default;

                    #endregion

                    #region Another Semantics

                // i'xxx'
                case Tokens.SingleQuotedIdentifier:
                    tokenSemantics.Object = (string)GetTokenAsSinglyQuotedString(1, encoding, false);
                    token = Tokens.T_STRING;
                    goto default;

                    #endregion

                    #region Token Reinterpreting

                case Tokens.T_OPEN_TAG_WITH_ECHO:
                    this.commentsSink.OnOpenTag(this, this.tokenPosition);
                    token = Tokens.T_ECHO;
                    goto default;

                case Tokens.T_CLOSE_TAG:
                    this.commentsSink.OnCloseTag(this, this.tokenPosition);
                    token = Tokens.T_SEMI;
                    goto case Tokens.T_SEMI;

                case Tokens.T_TRUE:
                case Tokens.T_FALSE:
                case Tokens.T_NULL:
                case Tokens.T_GET:
                case Tokens.T_SET:
                case Tokens.T_CALL:
                case Tokens.T_CALLSTATIC:
                case Tokens.T_WAKEUP:
                case Tokens.T_SLEEP:
                case Tokens.T_TOSTRING:
                case Tokens.T_CONSTRUCT:
                case Tokens.T_DESTRUCT:
                case Tokens.T_PARENT:
                case Tokens.T_SELF:
                case Tokens.T_AUTOLOAD:
                    token = Tokens.T_STRING;
                    goto case Tokens.T_STRING;

                case Tokens.T_TRY:
                case Tokens.T_CATCH:
                case Tokens.T_FINALLY:
                case Tokens.T_THROW:
                case Tokens.T_IMPLEMENTS:
                case Tokens.T_CLONE:
                case Tokens.T_ABSTRACT:
                case Tokens.T_FINAL:
                case Tokens.T_PRIVATE:
                case Tokens.T_PROTECTED:
                case Tokens.T_PUBLIC:
                case Tokens.T_INSTANCEOF:
                case Tokens.T_INTERFACE:
                case Tokens.T_GOTO:
                case Tokens.T_NAMESPACE:
                case Tokens.T_NAMESPACE_C:
                case Tokens.T_NS_SEPARATOR:
                case Tokens.T_USE:
                {
                    if ((features & LanguageFeatures.V5Keywords) == 0)
                    {
                        token = Tokens.T_STRING;
                        goto case Tokens.T_STRING;
                    }

                    if (token == Tokens.T_ABSTRACT)
                    {
                        // remember this for possible CLR qualified name:
                        tokenSemantics.Object = base.GetTokenString();
                    }

                    goto default;
                }

                case Tokens.T_LINQ_FROM:
                case Tokens.T_LINQ_SELECT:
                case Tokens.T_LINQ_BY:
                case Tokens.T_LINQ_WHERE:
                case Tokens.T_LINQ_DESCENDING:
                case Tokens.T_LINQ_ASCENDING:
                case Tokens.T_LINQ_ORDERBY:
                case Tokens.T_LINQ_GROUP:
                case Tokens.T_LINQ_IN:
                {
                    if ((features & LanguageFeatures.Linq) == 0)
                    {
                        token = Tokens.T_STRING;
                        goto case Tokens.T_STRING;
                    }

                    goto default;
                }

                case Tokens.T_IMPORT:
                {
                    if (!sourceUnit.CompilationUnit.IsPure)
                    {
                        token = Tokens.T_STRING;
                        goto case Tokens.T_STRING;
                    }

                    goto default;
                }

                case Tokens.T_BOOL_TYPE:
                case Tokens.T_INT_TYPE:
                case Tokens.T_INT64_TYPE:
                case Tokens.T_DOUBLE_TYPE:
                case Tokens.T_STRING_TYPE:
                case Tokens.T_RESOURCE_TYPE:
                case Tokens.T_OBJECT_TYPE:
                case Tokens.T_TYPEOF:
                {
                    if ((features & LanguageFeatures.TypeKeywords) == 0)
                    {
                        token = Tokens.T_STRING;
                        goto case Tokens.T_STRING;
                    }

                    tokenSemantics.Object = base.GetTokenString();

                    goto default;
                }

                case Tokens.T_PARTIAL:
                {
                    if (!pure)
                    {
                        token = Tokens.T_STRING;
                        goto case Tokens.T_STRING;
                    }

                    goto default;
                }

                    #endregion

                    #region Error Tokens

                case Tokens.ERROR:
                    goto default;

                case Tokens.ErrorInvalidIdentifier:
                {
                    // invalid identifier i'XXX':
                    errors.Add(Errors.InvalidIdentifier, SourceUnit, tokenPosition, (string)GetTokenAsSinglyQuotedString(1, encoding, false));

                    tokenSemantics.Object = GetErrorIdentifier();
                    token = Tokens.T_STRING;
                    goto default;
                }

                case Tokens.ErrorNotSupported:
                    errors.Add(Errors.ConstructNotSupported, SourceUnit, tokenPosition, GetTokenString());
                    tokenSemantics.Object = GetErrorIdentifier();
                    token = Tokens.T_STRING;
                    goto default;

                    #endregion

                case Tokens.T_SEMI:
                default:

                    if (lastDocComment != null)
                    {
                        // remember PHPDoc for current token
                        if (lastDocCommentRememberTokens.Contains(token))
                        {
                            tokenSemantics.Object = this.lastDocComment;
                        }

                        // forget last doc comment text
                        if (!lastDocCommentKeepTokens.Contains(token))
                        {
                            lastDocComment = null;
                        }
                    }

                    return(token);
                }
            }
        }