public bool AddDebugLookup(string debugNamespace)
        {
            // a blank identifier is okay -- we just ignore it
            if (!string.IsNullOrEmpty(debugNamespace))
            {
                // but if it's not blank, it better be a valid JavaScript identifier or member chain
                if (debugNamespace.IndexOf('.') > 0)
                {
                    // it's a member chain -- check that each part is a valid JS identifier
                    var names = debugNamespace.Split('.');
                    foreach (var name in names)
                    {
                        if (!JSScanner.IsValidIdentifier(name))
                        {
                            return(false);
                        }
                    }
                }
                else
                {
                    // no dot -- just an identifier
                    if (!JSScanner.IsValidIdentifier(debugNamespace))
                    {
                        return(false);
                    }
                }

                m_debugLookups.Add(debugNamespace);
                return(true);
            }

            return(false);
        }
Exemple #2
0
        public override string ToCode(ToCodeFormat format)
        {
            StringBuilder sb = new StringBuilder();

            sb.Append("do");

            ToCodeFormat bodyFormat = ((Body != null &&
                                        Body.Count == 1 &&
                                        Body[0].GetType() == typeof(DoWhile))
              ? ToCodeFormat.AlwaysBraces
              : ToCodeFormat.Normal
                                       );

            // if the body is a single statement that ends in a do-while, then we
            // will need to wrap the body in curly-braces to get around an IE bug
            if (Body != null && Body.EncloseBlock(EncloseBlockType.SingleDoWhile))
            {
                bodyFormat = ToCodeFormat.AlwaysBraces;
            }

            string bodyString = (
                Body == null
              ? string.Empty
              : Body.ToCode(bodyFormat)
                );

            if (bodyString.Length == 0)
            {
                sb.Append(';');
            }
            else
            {
                // if the first character could be interpreted as a continuation
                // of the "do" keyword, then we need to add a space
                if (JSScanner.StartsWithIdentifierPart(bodyString))
                {
                    sb.Append(' ');
                }
                sb.Append(bodyString);

                // if there is no body, we need a semi-colon
                // OR if we didn't always wrap in braces AND we require a separator, we need a semi-colon.
                // and make sure it doesn't already end in a semicolon -- we don't want two in a row.
                if (Body == null ||
                    (bodyFormat != ToCodeFormat.AlwaysBraces && Body.RequiresSeparator && !bodyString.EndsWith(";", StringComparison.Ordinal)))
                {
                    sb.Append(';');
                }
            }
            // add a space for readability for pretty-print mode
            if (Parser.Settings.OutputMode == OutputMode.MultipleLines && Parser.Settings.IndentSize > 0)
            {
                sb.Append(' ');
            }
            sb.Append("while(");
            sb.Append(Condition.ToCode());
            sb.Append(")");
            return(sb.ToString());
        }
Exemple #3
0
        /// <summary>
        /// Add a known global identifier to the list
        /// </summary>
        /// <param name="identifier">global identifier</param>
        /// <returns>true if valid identifier; false if invalid identifier</returns>
        public bool AddKnownGlobal(string identifier)
        {
            if (JSScanner.IsValidIdentifier(identifier))
            {
                m_knownGlobals.Add(identifier);
                return(true);
            }

            return(false);
        }
Exemple #4
0
        public bool AddNoAutoRename(string noRename)
        {
            if (!JSScanner.IsValidIdentifier(noRename))
            {
                return(false);
            }

            m_noRenameSet.Add(noRename);
            return(true);
        }
 private JSScanner GetJSScanner()
 {
     if (_scanner == null)
     {
         _scanner = new JSScanner(new CodeSettings()
         {
             AllowShebangLine = true
         });
     }
     return(_scanner);
 }
Exemple #6
0
        private static List <TokenWithSpan> ReadTokens(string code, bool collectWarnings, params ErrorInfo[] errors)
        {
            CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings);
            var scanner = new JSScanner(code, errorSink, new CodeSettings()
            {
                AllowShebangLine = true
            });
            var tokens = scanner.ReadTokens(Int32.MaxValue);

            errorSink.CheckErrors(errors);
            return(tokens);
        }
Exemple #7
0
        internal string NextName()
        {
            string name;

            do
            {
                // advance to the next name
                ++m_currentName;
                name = CurrentName;
                // keep advancing until we find one that isn't in the skip list or a keyword
            }while (m_skipNames.ContainsKey(name) || JSScanner.IsKeyword(name));
            return(name);
        }
Exemple #8
0
        // Determine if an identifier is valid for this engine.
        public override bool IsValidIdentifier(String identifier)
        {
            if (identifier == null)
            {
                return(false);
            }
            JSScanner scanner = new JSScanner(identifier);

            if (scanner.FetchIdentifier() == null)
            {
                return(false);
            }
            return(scanner.Fetch() == -1);
        }
Exemple #9
0
        public override string ToCode(ToCodeFormat format)
        {
            StringBuilder sb = new StringBuilder();

            sb.Append("return");
            if (m_operand != null)
            {
                string operandText = m_operand.ToCode();
                if (operandText.Length > 0 && JSScanner.StartsWithIdentifierPart(operandText))
                {
                    sb.Append(' ');
                }
                sb.Append(operandText);
            }
            return(sb.ToString());
        }
Exemple #10
0
        public override string ToCode(ToCodeFormat format)
        {
            if (PrimitiveType == PrimitiveType.String)
            {
                string rawValue = Value.ToString();

                // if the raw value is safe to be an identifier, then go ahead and ditch the quotes and just output
                // the raw value. Otherwise call ToCode to wrap the string in quotes.
                return(JSScanner.IsSafeIdentifier(rawValue) && !JSScanner.IsKeyword(rawValue) ? rawValue : base.ToCode(format));
            }
            else
            {
                // call the base to format the value
                return(base.ToCode(format));
            }
        }
Exemple #11
0
        /// <summary>
        /// Set the dictionary of preprocessor defines and values
        /// </summary>
        /// <param name="defines">dictionary to set</param>
        public int SetPreprocessorValues(IDictionary <string, string> defines)
        {
            PreprocessorValues.Clear();
            if (defines != null && defines.Count > 0)
            {
                foreach (var define in defines)
                {
                    if (JSScanner.IsValidIdentifier(define.Key))
                    {
                        PreprocessorValues.Add(define.Key, define.Value);
                    }
                }
            }

            return(PreprocessorValues.Count);
        }
Exemple #12
0
        private static bool NeedsParens(AstNode node, JSToken refToken)
        {
            bool needsParens = false;

            // assignments and commas are the only operators that need parens
            // around them. Conditional is pretty low down the list
            BinaryOperator binaryOp = node as BinaryOperator;

            if (binaryOp != null)
            {
                OpPrec thisPrecedence = JSScanner.GetOperatorPrecedence(refToken);
                OpPrec nodePrecedence = JSScanner.GetOperatorPrecedence(binaryOp.OperatorToken);
                needsParens = (nodePrecedence < thisPrecedence);
            }

            return(needsParens);
        }
        /// <summary>
        /// Rescans the part of the buffer affected by a change.
        /// Scans a contiguous sub-<paramref name="span"/> of a larger code span which starts at <paramref name="codeStartLine"/>.
        /// </summary>
        private void ApplyChange(JSScanner JSScanner, ITextSnapshot snapshot, Span span)
        {
            int firstLine = snapshot.GetLineNumberFromPosition(span.Start);
            int lastLine  = snapshot.GetLineNumberFromPosition(span.Length > 0 ? span.End - 1 : span.End);

            Contract.Assert(firstLine >= 0);

            // find the closest line preceding firstLine for which we know categorizer state, stop at the codeStartLine:
            LineTokenization lineTokenization;

            firstLine = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenization) + 1;
            object state = lineTokenization.State;

            int    currentLine = firstLine;
            object previousState;

            while (currentLine < snapshot.LineCount)
            {
                previousState            = _tokenCache.TryGetTokenization(currentLine, out lineTokenization) ? lineTokenization.State : null;
                _tokenCache[currentLine] = lineTokenization = TokenizeLine(JSScanner, snapshot, state, currentLine);
                state = lineTokenization.State;

                // stop if we visted all affected lines and the current line has no tokenization state or its previous state is the same as the new state:
                if (currentLine > lastLine && (previousState == null || previousState.Equals(state)))
                {
                    break;
                }

                currentLine++;
            }

            // classification spans might have changed between the start of the first and end of the last visited line:
            int changeStart = snapshot.GetLineFromLineNumber(firstLine).Start;
            int changeEnd   = (currentLine < snapshot.LineCount) ? snapshot.GetLineFromLineNumber(currentLine).End : snapshot.Length;

            if (changeStart < changeEnd)
            {
                var classificationChanged = ClassificationChanged;
                if (classificationChanged != null)
                {
                    var args = new ClassificationChangedEventArgs(new SnapshotSpan(snapshot, new Span(changeStart, changeEnd - changeStart)));
                    classificationChanged(this, args);
                }
            }
        }
Exemple #14
0
        private static List <TokenWithSpan> ScanTokens(string code, bool collectWarnings, params ErrorInfo[] errors)
        {
            CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings);
            var scanner = new JSScanner(code, errorSink, new CodeSettings()
            {
                AllowShebangLine = true
            });
            List <TokenWithSpan> tokens = new List <TokenWithSpan>();

            for (TokenWithSpan curToken = scanner.ScanNextTokenWithSpan(true);
                 curToken.Token != JSToken.EndOfFile;
                 curToken = scanner.ScanNextTokenWithSpan(true))
            {
                tokens.Add(curToken);
            }
            errorSink.CheckErrors(errors);
            return(tokens);
        }
Exemple #15
0
    // Test a single operator.
    private void TestOp(JSScanner scanner, String ops,
                        ref int posn, JSToken expected)
    {
        Context token = JSScannerTest.TestGetTokenContext(scanner);

        scanner.GetNextToken();
        int    next   = ops.IndexOf(' ', posn);
        String thisop = ops.Substring(posn, next - posn);

        AssertEquals("TestOp[" + thisop + "] (1)",
                     expected, token.GetToken());
        AssertEquals("TestOp[" + thisop + "] (2)",
                     thisop, token.GetCode());
        AssertEquals("TestOp[" + thisop + "] (3)",
                     posn, token.StartPosition);
        AssertEquals("TestOp[" + thisop + "] (4)",
                     next, token.EndPosition);
        posn = next + 1;
    }
        private LineTokenization TokenizeLine(JSScanner JSScanner, ITextSnapshot snapshot, object previousLineState, int lineNo)
        {
            ITextSnapshotLine line     = snapshot.GetLineFromLineNumber(lineNo);
            SnapshotSpan      lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak);

            var tcp = new SnapshotSpanSourceCodeReader(lineSpan);

            JSScanner.Initialize(
                lineSpan.GetText(),
                previousLineState,
                new SourceLocation(0, lineNo + 1, 1)
                );
            try {
                var tokens = JSScanner.ReadTokens(lineSpan.Length).Select(ToTokenKind).ToArray();
                return(new LineTokenization(tokens, JSScanner.CurrentState));
            } finally {
                JSScanner.Uninitialize();
            }
        }
        public static string CrunchedLabel(int nestLevel)
        {
            // nestCount is 0-based.
            // return null if the nestLevel is invalid (< 0).
            string minLabel = null;

            if (nestLevel >= 0)
            {
                minLabel = GenerateNameFromNumber(nestLevel);
                if (JSScanner.IsKeyword(minLabel, true))
                {
                    // prepend something to make it NOT a keyword.
                    // no keywords start with an underscore, so...
                    minLabel = '_' + minLabel;
                }
            }

            return(minLabel);
        }
Exemple #18
0
        private static bool IsOnlyDecimalDigits(string text)
        {
            // if text is null, return false.
            // Otherwise return true if ALL the characters are decimal digits,
            // or false is ANY ONE character isn't.
            //return text.IfNotNull(s => !s.Any(c => !JSScanner.IsDigit(c)));

            return(text.IfNotNull(s =>
            {
                foreach (char c in s)
                {
                    if (!JSScanner.IsDigit(c))
                    {
                        return false;
                    }
                }
                return true;
            }));
        }
Exemple #19
0
        public override string ToCode(ToCodeFormat format)
        {
            string operandString = Operand.ToCode(format);

            if (NeedsParentheses)
            {
                // needs parens
                return("delete(" + operandString + ')');
            }
            else if (JSScanner.StartsWithIdentifierPart(operandString))
            {
                // needs a space
                return("delete " + operandString);
            }
            else
            {
                // needs no separator
                return("delete" + operandString);
            }
        }
Exemple #20
0
        public override string ToCode(ToCodeFormat format)
        {
            string operandString = Operand.ToCode(format);

            if (NeedsParentheses)
            {
                // need parentheses
                return("typeof(" + operandString + ')');
            }
            else if (JSScanner.StartsWithIdentifierPart(operandString))
            {
                // need a space separating them
                return("typeof " + operandString);
            }
            else
            {
                // don't need the space
                return("typeof" + operandString);
            }
        }
Exemple #21
0
        public override string ToCode(ToCodeFormat format)
        {
            StringBuilder sb = new StringBuilder();

            // the label should be indented
            Parser.Settings.Indent();
            // start a new line
            Parser.Settings.NewLine(sb);
            if (m_caseValue != null)
            {
                sb.Append("case");

                string caseValue = m_caseValue.ToCode();
                if (JSScanner.StartsWithIdentifierPart(caseValue))
                {
                    sb.Append(' ');
                }
                sb.Append(caseValue);
            }
            else
            {
                sb.Append("default");
            }
            sb.Append(':');

            // in pretty-print mode, we indent the statements under the label, too
            Parser.Settings.Indent();

            // output the statements
            if (m_statements != null && m_statements.Count > 0)
            {
                sb.Append(m_statements.ToCode(ToCodeFormat.NoBraces));
            }

            // if we are pretty-printing, we need to unindent twice:
            // once for the label, and again for the statements
            Parser.Settings.Unindent();
            Parser.Settings.Unindent();
            return(sb.ToString());
        }
        private void UnreferencedFunction(JSVariableField variableField, FunctionObject functionObject)
        {
            // if there is no name, then ignore this declaration because it's malformed.
            // (won't be a function expression because those are automatically referenced).
            // also ignore ghosted function fields.
            if (functionObject.Name != null && variableField.FieldType != FieldType.GhostFunction)
            {
                // if the function name isn't a simple identifier, then leave it there and mark it as
                // not renamable because it's probably one of those darn IE-extension event handlers or something.
                if (JSScanner.IsValidIdentifier(functionObject.Name))
                {
                    // unreferenced function declaration. fire a warning.
                    var ctx = functionObject.IdContext ?? variableField.OriginalContext;
                    ctx.HandleError(JSError.FunctionNotReferenced, false);

                    // hide it from the output if our settings say we can.
                    // we don't want to delete it, per se, because we still want it to
                    // show up in the scope report so the user can see that it was unreachable
                    // in case they are wondering where it went.
                    // ES6 has the notion of block-scoped function declarations. ES5 says functions can't
                    // be defined inside blocks -- only at the root level of the global scope or function scopes.
                    // so if this is a block scope, don't hide the function, even if it is unreferenced because
                    // of the cross-browser difference.
                    if (this.IsKnownAtCompileTime &&
                        m_settings.MinifyCode &&
                        m_settings.RemoveUnneededCode &&
                        !(this is BlockScope))
                    {
                        functionObject.HideFromOutput = true;
                    }
                }
                else
                {
                    // not a valid identifier name for this function. Don't rename it because it's
                    // malformed and we don't want to mess up the developer's intent.
                    variableField.CanCrunch = false;
                }
            }
        }
Exemple #23
0
        // Determine if a namespace name is valid.
        protected override bool IsValidNamespaceName(String name)
        {
            if (name == null)
            {
                return(false);
            }
            JSScanner scanner = new JSScanner(name);

            if (scanner.FetchIdentifier() == null)
            {
                return(false);
            }
            while (scanner.Peek() == '.')
            {
                scanner.Fetch();
                if (scanner.FetchIdentifier() == null)
                {
                    return(false);
                }
            }
            return(scanner.Fetch() == -1);
        }
        public void PartialScanning() {
            var code1 = "/* hello world ";
            var code2 = "   goodbye */";
            CollectingErrorSink errorSink = new CollectingErrorSink(true);
            var scanner = new JSScanner(code1, errorSink, new CodeSettings() { AllowShebangLine = true });
            var tokens = scanner.ReadTokens(Int32.MaxValue);
            VerifyTokens(
                tokens,
                new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(0, 1, 1), new SourceLocation(15, 1, 16))
            );

            scanner.Initialize(code2, scanner.CurrentState, new SourceLocation(code1.Length, 2, 1));

            tokens = scanner.ReadTokens(Int32.MaxValue);
            VerifyTokens(
                tokens,
                new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(15, 2, 16), new SourceLocation(28, 2, 29))
            );

            Assert.IsTrue(scanner.CurrentState.Equals(scanner.CurrentState));
            Assert.IsFalse(scanner.CurrentState.Equals(new object()));
            Assert.AreEqual(scanner.CurrentState.GetHashCode(), 2);
        }
Exemple #25
0
 /// <summary>
 /// We really only care about caching strings that are likely
 /// to be identifiers.  Everything else we'll coalesce into the
 /// empty string.
 /// </summary>
 private static bool DontCacheString(string strValue)
 {
     if (strValue.Length > 100)
     {
         return(true);
     }
     for (int i = 0; i < strValue.Length; i++)
     {
         char ch = strValue[i];
         if (ch == '-' || ch == '.' || ch == '/' || ch == '\\')
         {
             // not valid identifiers, but likely to show up in
             // require string literals, so we still care about preserving
             // these strings.
             continue;
         }
         if (!JSScanner.IsValidIdentifierPart(strValue[i]))
         {
             return(true);
         }
     }
     return(false);
 }
        public bool Match(AstNode node, string identifiers)
        {
            // set the match to false
            m_isMatch = false;

            // identifiers cannot be null or blank and must match: IDENT(.IDENT)*
            // since for JS there has to be at least a global object, the dot must be AFTER the first character.
            if (node != null && !string.IsNullOrEmpty(identifiers))
            {
                // get all the parts
                var parts = identifiers.Split('.');

                // each part must be a valid JavaScript identifier. Assume everything is valid
                // unless at least one is invalid -- then forget it
                var isValid = true;
                foreach (var part in parts)
                {
                    if (!JSScanner.IsValidIdentifier(part))
                    {
                        isValid = false;
                        break;
                    }
                }

                // must be valid to continue
                if (isValid)
                {
                    // save the parts and start the index on the last one, since we'll be walking backwards
                    m_parts = parts;
                    m_index = parts.Length - 1;

                    node.Accept(this);
                }
            }

            return(m_isMatch);
        }
Exemple #27
0
        /// <summary>
        /// Add a rename pair to the identifier rename map
        /// </summary>
        /// <param name="sourceName">name of the identifier in the source code</param>
        /// <param name="newName">new name with which to replace the source name</param>
        /// <returns>true if added; false if either name is not a valid JavaScript identifier</returns>
        public bool AddRenamePair(string sourceName, string newName)
        {
            bool successfullyAdded = false;

            // both names MUST be valid JavaScript identifiers
            if (JSScanner.IsValidIdentifier(sourceName) && JSScanner.IsValidIdentifier(newName))
            {
                if (m_identifierReplacementMap.ContainsKey(sourceName))
                {
                    // just replace the value
                    m_identifierReplacementMap[sourceName] = newName;
                }
                else
                {
                    // add the new pair
                    m_identifierReplacementMap.Add(sourceName, newName);
                }

                // if we get here, we added it (or updated it if it's a dupe)
                successfullyAdded = true;
            }

            return(successfullyAdded);
        }
        private LineTokenization GetPreviousTokenization(JSScanner JSScanner, ITextSnapshot snapshot, int firstLine, int prevLine) {
            LineTokenization prevLineTokenization;
            if (!_tokenCache.TryGetTokenization(prevLine, out prevLineTokenization)) {
                LineTokenization lineTokenizationTemp;
                int currentLineTemp = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenizationTemp) + 1;
                object stateTemp = lineTokenizationTemp.State;

                while (currentLineTemp < snapshot.LineCount) {
                    if (!_tokenCache.TryGetTokenization(currentLineTemp, out lineTokenizationTemp)) {
                        lineTokenizationTemp = TokenizeLine(JSScanner, snapshot, stateTemp, currentLineTemp);
                        _tokenCache[currentLineTemp] = lineTokenizationTemp;
                    }

                    currentLineTemp++;
                    stateTemp = lineTokenizationTemp.State;
                }

                prevLineTokenization = TokenizeLine(JSScanner, snapshot, stateTemp, prevLine);
                _tokenCache[prevLine] = prevLineTokenization;
            }
            return prevLineTokenization;
        }
Exemple #29
0
        static string CreateJSFromResourceStrings(ResourceStrings resourceStrings)
        {
            var sb = StringBuilderPool.Acquire();

            try
            {
                // start the var statement using the requested name and open the initializer object literal
                sb.Append("var ");
                sb.Append(resourceStrings.Name);
                sb.Append("={");

                // we're going to need to insert commas between each pair, so we'll use a boolean
                // flag to indicate that we're on the first pair. When we output the first pair, we'll
                // set the flag to false. When the flag is false, we're about to insert another pair, so
                // we'll add the comma just before.
                bool firstItem = true;

                // loop through all items in the collection
                foreach (var keyPair in resourceStrings.NameValuePairs)
                {
                    // if this isn't the first item, we need to add a comma separator
                    if (!firstItem)
                    {
                        sb.Append(',');
                    }
                    else
                    {
                        // next loop is no longer the first item
                        firstItem = false;
                    }

                    // append the key as the name, a colon to separate the name and value,
                    // and then the value
                    // must quote if not valid JS identifier format, or if it is, but it's a keyword
                    // (use strict mode just to be safe)
                    string propertyName = keyPair.Key;
                    if (!JSScanner.IsValidIdentifier(propertyName) || JSScanner.IsKeyword(propertyName, true))
                    {
                        sb.Append("\"");
                        // because we are using quotes for the delimiters, replace any instances
                        // of a quote character (") with an escaped quote character (\")
                        sb.Append(propertyName.Replace("\"", "\\\""));
                        sb.Append("\"");
                    }
                    else
                    {
                        sb.Append(propertyName);
                    }
                    sb.Append(':');

                    // make sure the Value is properly escaped, quoted, and whatever we
                    // need to do to make sure it's a proper JS string.
                    // pass false for whether this string is an argument to a RegExp constructor.
                    // pass false for whether to use W3Strict formatting for character escapes (use maximum browser compatibility)
                    // pass true for ecma strict mode
                    string stringValue = ConstantWrapper.EscapeString(
                        keyPair.Value,
                        false,
                        false,
                        true
                        );
                    sb.Append(stringValue);
                }

                // close the object literal and return the string
                sb.AppendLine("};");
                return(sb.ToString());
            }
            finally
            {
                sb.Release();
            }
        }
        /// <summary>
        /// Creates an instance of the JSParser class that can be used to parse the given source code.
        /// </summary>
        /// <param name="source">Source code to parse.</param>
        public JSParser(string source)
        {
            m_severity = 5;
            m_blockType = new List<BlockType>(16);
            m_labelTable = new Dictionary<string, LabelInfo>();
            m_noSkipTokenSet = new NoSkipTokenSet();
            m_importantComments = new List<Context>();

            m_document = new DocumentContext(this, source);
            m_scanner = new JSScanner(new Context(m_document));
            m_currentToken = new Context(m_document);

            // if the scanner encounters a special "globals" comment, it'll fire this event
            // at which point we will define a field with that name in the global scope. 
            m_scanner.GlobalDefine += (sender, ea) =>
                {
                    var globalScope = GlobalScope;
                    if (globalScope[ea.Name] == null)
                    {
                        var field = globalScope.CreateField(ea.Name, null, FieldAttributes.SpecialName);
                        globalScope.AddField(field);
                    }
                };

            // this event is fired whenever a ///#SOURCE comment is encountered
            m_scanner.NewModule += (sender, ea) =>
                {
                    m_newModule = true;
                };
        }
	// Test a single operator.
	private void TestOp(JSScanner scanner, String ops,
					    ref int posn, JSToken expected)
			{
				Context token = JSScannerTest.TestGetTokenContext(scanner);
				scanner.GetNextToken();
				int next = ops.IndexOf(' ', posn);
				String thisop = ops.Substring(posn, next - posn);
				AssertEquals("TestOp[" + thisop + "] (1)",
							 expected, token.GetToken());
				AssertEquals("TestOp[" + thisop + "] (2)",
							 thisop, token.GetCode());
				AssertEquals("TestOp[" + thisop + "] (3)",
							 posn, token.StartPosition);
				AssertEquals("TestOp[" + thisop + "] (4)",
							 next, token.EndPosition);
				posn = next + 1;
			}
 private static List<TokenWithSpan> ScanTokens(string code, bool collectWarnings, params ErrorInfo[] errors) {
     CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings);
     var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true });
     List<TokenWithSpan> tokens = new List<TokenWithSpan>();
     for (TokenWithSpan curToken = scanner.ScanNextTokenWithSpan(true);
         curToken.Token != JSToken.EndOfFile;
         curToken = scanner.ScanNextTokenWithSpan(true)) {
         tokens.Add(curToken);
     }
     errorSink.CheckErrors(errors);
     return tokens;
 }
 private static List<TokenWithSpan> ReadTokens(string code, bool collectWarnings, params ErrorInfo[] errors) {
     CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings);
     var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true });
     var tokens = scanner.ReadTokens(Int32.MaxValue);
     errorSink.CheckErrors(errors);
     return tokens;
 }
Exemple #34
0
        internal override Dictionary <string, IAnalysisSet> GetOwnProperties(ProjectEntry accessor)
        {
            if (_descriptors == null || _descriptors.Count == 0)
            {
                return(new Dictionary <string, IAnalysisSet>());
            }

            var res = new Dictionary <string, IAnalysisSet>();

            foreach (var kvp in _descriptors)
            {
                var key = kvp.Key;
                if (!JSScanner.IsValidIdentifier(key))
                {
                    // https://nodejstools.codeplex.com/workitem/987
                    // for intellisense purposes we don't report invalid identifiers, but we can
                    // end up with these from indexing with constant strings.
                    continue;
                }

                if (kvp.Value.Values != null)
                {
                    //kvp.Value.Values.ClearOldValues();
                    if (kvp.Value.Values.VariableStillExists)
                    {
                        var types = kvp.Value.Values.GetTypesNoCopy(accessor, ProjectEntry);
                        if (types.Count != 0 ||
                            kvp.Value.Values.TypesNoCopy.Count == 0)
                        {
                            MergeTypes(res, key, types);
                        }
                    }
                }

                if (kvp.Value.Getter != null)
                {
                    foreach (var value in kvp.Value.Getter.GetTypesNoCopy(accessor, ProjectEntry))
                    {
                        FunctionValue userFunc = value.Value as FunctionValue;
                        if (userFunc != null)
                        {
                            MergeTypes(res, key, userFunc.ReturnTypes);
                        }
                    }
                }

                if (kvp.Value.Setter != null)
                {
                    MergeTypes(res, key, AnalysisSet.Empty);
                }
            }

            if (_linkedValues != null)
            {
                foreach (var linkedValue in _linkedValues.GetTypesNoCopy(accessor, ProjectEntry))
                {
                    if (linkedValue.Value.Push())
                    {
                        try {
                            MergeDictionaries(res, linkedValue.Value.GetAllMembers(accessor));
                        } finally {
                            linkedValue.Value.Pop();
                        }
                    }
                }
            }

            return(res);
        }
        /// <summary>
        /// Adds classification spans to the given collection.
        /// Scans a contiguous sub-<paramref name="span"/> of a larger code span which starts at <paramref name="codeStartLine"/>.
        /// </summary>
        private void AddClassifications(JSScanner JSScanner, List<ClassificationSpan> classifications, SnapshotSpan span) {
            Debug.Assert(span.Length > 0);

            var snapshot = span.Snapshot;
            int firstLine = snapshot.GetLineNumberFromPosition(span.Start);
            int lastLine = snapshot.GetLineNumberFromPosition(span.End - 1);

            Contract.Assert(firstLine >= 0);

            _tokenCache.EnsureCapacity(snapshot.LineCount);

            // find the closest line preceding firstLine for which we know categorizer state, stop at the codeStartLine:
            LineTokenization lineTokenization;
            int currentLine = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenization) + 1;
            object state = lineTokenization.State;

            // track the previous 2 tokens to adjust our classifications of keywords
            // when they shouldn't be displayed as keywords...
            TokenInfoWithLine? prevToken = null, prevPrevToken = null;

            // initialize the previous tokens so we can handle things like:
            //      foo.
            //          get()
            // even if we're called on the line for get()
            int prevLine = currentLine - 1;
            while (prevLine >= 0 && prevToken == null) {
                LineTokenization prevLineTokenization = GetPreviousTokenization(JSScanner, snapshot, firstLine, prevLine);
                for (int i = prevLineTokenization.Tokens.Length - 1; i >= 0 && prevToken == null; i--) {
                    var tempToken = prevLineTokenization.Tokens[i];
                    if (IsValidPreviousToken(ref tempToken)) {
                        prevToken = prevPrevToken;
                        prevPrevToken = new TokenInfoWithLine() { TokenInfo = tempToken, Line = prevLine };
                    }
                }
                prevLine--;
            }

            while (currentLine <= lastLine) {
                if (!_tokenCache.TryGetTokenization(currentLine, out lineTokenization)) {
                    lineTokenization = TokenizeLine(JSScanner, snapshot, state, currentLine);
                    _tokenCache[currentLine] = lineTokenization;
                }

                state = lineTokenization.State;

                for (int i = 0; i < lineTokenization.Tokens.Length; i++) {
                    var token = lineTokenization.Tokens[i];

                    if (token.Category == TokenCategory.IncompleteMultiLineStringLiteral || token.Category == TokenCategory.Comment) {
                        IClassificationType type;
                        switch (token.Category) {
                            case TokenCategory.IncompleteMultiLineStringLiteral:
                                type = _provider.StringLiteral;
                                break;
                            case TokenCategory.Comment:
                                type = _provider.Comment;
                                break;
                            default:
                                type = null;
                                break;
                        }

                        Debug.Assert(type != null, "We should have a defined ClassificationType for every token.");

                        // we need to walk backwards to find the start of this multi-line string...
                        TokenInfo startToken = token;
                        int validPrevLine;
                        int length = startToken.SourceSpan.Length;
                        if (i == 0) {
                            length += GetLeadingMultiLineTokens(JSScanner, snapshot, token.Category, firstLine, currentLine, out validPrevLine, ref startToken);
                        } else {
                            validPrevLine = currentLine;
                        }

                        if (i == lineTokenization.Tokens.Length - 1) {
                            length += GetTrailingMultiLineTokens(JSScanner, snapshot, token.Category, currentLine, state);
                        }

                        var tokenSpan = new Span(SnapshotSpanToSpan(snapshot, startToken, validPrevLine).Start, length);
                        var intersection = span.Intersection(tokenSpan);

                        if ((intersection != null && intersection.Value.Length > 0) ||
                            (span.Length == 0 && tokenSpan.Contains(span.Start)) // handle zero-length spans
                        ) {
                            classifications.Add(new ClassificationSpan(new SnapshotSpan(snapshot, tokenSpan), type));
                        }
                    } else {
                        ClassificationSpan classification = null;
                        if (token.Category == TokenCategory.Keyword) {
                            // check and see if we're not really a keyword...
                            if (IsKeywordInIdentifierContext(snapshot, prevToken, prevPrevToken, new TokenInfoWithLine() { TokenInfo = token, Line = currentLine })) {
                                classification = GetClassificationSpan(
                                    span,
                                    token,
                                    currentLine,
                                    CategoryMap[TokenCategory.Identifier]
                                );
                            }
                        }
                        if (classification == null) {
                            classification = ClassifyToken(span, token, currentLine);
                        }

                        if (classification != null) {
                            classifications.Add(classification);
                        }
                    }

                    if (IsValidPreviousToken(ref token)) {
                        prevPrevToken = prevToken;
                        prevToken = new TokenInfoWithLine() { TokenInfo = token, Line = currentLine };
                    }
                }

                currentLine++;
            }
        }
        private int GetLeadingMultiLineTokens(JSScanner JSScanner, ITextSnapshot snapshot, TokenCategory tokenCategory, int firstLine, int currentLine, out int validPrevLine, ref TokenInfo startToken) {
            validPrevLine = currentLine;
            int prevLine = currentLine - 1;
            int length = 0;

            while (prevLine >= 0) {
                LineTokenization prevLineTokenization = GetPreviousTokenization(JSScanner, snapshot, firstLine, prevLine);

                if (prevLineTokenization.Tokens.Length != 0) {
                    if (prevLineTokenization.Tokens[prevLineTokenization.Tokens.Length - 1].Category != tokenCategory) {
                        break;
                    }

                    startToken = prevLineTokenization.Tokens[prevLineTokenization.Tokens.Length - 1];
                    length += startToken.SourceSpan.Length;
                }

                validPrevLine = prevLine;
                prevLine--;

                if (prevLineTokenization.Tokens.Length > 1) {
                    // http://pytools.codeplex.com/workitem/749
                    // if there are multiple tokens on this line then our multi-line string
                    // is terminated.
                    break;
                }
            }
            return length;
        }
Exemple #37
0
        public override string ToCode(ToCodeFormat format)
        {
            StringBuilder sb = new StringBuilder();

            sb.Append("if(");
            sb.Append(Condition.ToCode());
            sb.Append(')');

            // if we're in Safari-quirks mode, we will need to wrap the if block
            // in curly braces if it only includes a function declaration. Safari
            // throws parsing errors in those situations
            ToCodeFormat elseFormat = ToCodeFormat.Normal;

            if (FalseBlock != null && FalseBlock.Count == 1)
            {
                if (Parser.Settings.MacSafariQuirks &&
                    FalseBlock[0] is FunctionObject)
                {
                    elseFormat = ToCodeFormat.AlwaysBraces;
                }
                else if (FalseBlock[0] is IfNode)
                {
                    elseFormat = ToCodeFormat.ElseIf;
                }
            }

            // get the else block -- we need to know if there is anything in order
            // to fully determine if the true-branch needs curly-braces
            string elseBlock = (
                FalseBlock == null
                ? string.Empty
                : FalseBlock.ToCode(elseFormat));

            // we'll need to force the true block to be enclosed in curly braces if
            // there is an else block and the true block contains a single statement
            // that ends in an if that doesn't have an else block
            ToCodeFormat trueFormat = (FalseBlock != null &&
                                       TrueBlock != null &&
                                       TrueBlock.EncloseBlock(EncloseBlockType.IfWithoutElse)
                ? ToCodeFormat.AlwaysBraces
                : ToCodeFormat.Normal);

            if (elseBlock.Length > 0 &&
                TrueBlock != null &&
                TrueBlock.EncloseBlock(EncloseBlockType.SingleDoWhile))
            {
                trueFormat = ToCodeFormat.AlwaysBraces;
            }

            // if we're in Safari-quirks mode, we will need to wrap the if block
            // in curly braces if it only includes a function declaration. Safari
            // throws parsing errors in those situations
            if (Parser.Settings.MacSafariQuirks &&
                TrueBlock != null &&
                TrueBlock.Count == 1 &&
                TrueBlock[0] is FunctionObject)
            {
                trueFormat = ToCodeFormat.AlwaysBraces;
            }

            // add the true block
            string trueBlock = (
                TrueBlock == null
                ? string.Empty
                : TrueBlock.ToCode(trueFormat));

            sb.Append(trueBlock);

            if (elseBlock.Length > 0)
            {
                if (trueFormat != ToCodeFormat.AlwaysBraces &&
                    !trueBlock.EndsWith(";", StringComparison.Ordinal) &&
                    (TrueBlock == null || TrueBlock.RequiresSeparator))
                {
                    sb.Append(';');
                }

                // if we are in pretty-print mode, drop the else onto a new line
                Parser.Settings.NewLine(sb);
                sb.Append("else");
                // if the first character could be interpreted as a continuation
                // of the "else" statement, then we need to add a space
                if (JSScanner.StartsWithIdentifierPart(elseBlock))
                {
                    sb.Append(' ');
                }

                sb.Append(elseBlock);
            }
            return(sb.ToString());
        }
        private int GetTrailingMultiLineTokens(JSScanner JSScanner, ITextSnapshot snapshot, TokenCategory tokenCategory, int currentLine, object state) {
            int nextLine = currentLine + 1;
            var prevState = state;
            int length = 0;
            while (nextLine < snapshot.LineCount) {
                LineTokenization nextLineTokenization;
                if (!_tokenCache.TryGetTokenization(nextLine, out nextLineTokenization)) {
                    nextLineTokenization = TokenizeLine(JSScanner, snapshot, prevState, nextLine);
                    prevState = nextLineTokenization.State;
                    _tokenCache[nextLine] = nextLineTokenization;
                }

                if (nextLineTokenization.Tokens.Length != 0) {
                    if (nextLineTokenization.Tokens[0].Category != tokenCategory) {
                        break;
                    }

                    length += nextLineTokenization.Tokens[0].SourceSpan.Length;
                }
                nextLine++;
            }
            return length;
        }
Exemple #39
0
        internal override void AnalyzeNode()
        {
            // see if this is a member (we'll need it for a couple checks)
            Member member = m_func as Member;

            if (Parser.Settings.StripDebugStatements && Parser.Settings.IsModificationAllowed(TreeModifications.StripDebugStatements))
            {
                // if this is a member, and it's a debugger object, and it's a constructor....
                if (member != null && member.IsDebuggerStatement && m_isConstructor)
                {
                    // we need to replace our debugger object with a generic Object
                    m_func = new Lookup("Object", m_func.Context, Parser);
                    // and make sure the node list is empty
                    if (m_args != null && m_args.Count > 0)
                    {
                        m_args = new AstNodeList(m_args.Context, Parser);
                    }
                }
            }

            // if this is a constructor and we want to collapse
            // some of them to literals...
            if (m_isConstructor && Parser.Settings.CollapseToLiteral)
            {
                // see if this is a lookup, and if so, if it's pointing to one
                // of the two constructors we want to collapse
                Lookup lookup = m_func as Lookup;
                if (lookup != null)
                {
                    if (lookup.Name == "Object" &&
                        Parser.Settings.IsModificationAllowed(TreeModifications.NewObjectToObjectLiteral))
                    {
                        // no arguments -- the Object constructor with no arguments is the exact same as an empty
                        // object literal
                        if (m_args == null || m_args.Count == 0)
                        {
                            // replace our node with an object literal
                            ObjectLiteral objLiteral = new ObjectLiteral(Context, Parser, null, null);
                            if (Parent.ReplaceChild(this, objLiteral))
                            {
                                // and bail now. No need to recurse -- it's an empty literal
                                return;
                            }
                        }
                        else if (m_args.Count == 1)
                        {
                            // one argument
                            // check to see if it's an object literal.
                            ObjectLiteral objectLiteral = m_args[0] as ObjectLiteral;
                            if (objectLiteral != null)
                            {
                                // the Object constructor with an argument that is a JavaScript object merely returns the
                                // argument. Since the argument is an object literal, it is by definition a JavaScript object
                                // and therefore we can replace the constructor call with the object literal
                                Parent.ReplaceChild(this, objectLiteral);

                                // don't forget to recurse the object now
                                objectLiteral.AnalyzeNode();

                                // and then bail -- we don't want to process this call
                                // operation any more; we've gotten rid of it
                                return;
                            }
                        }
                    }
                    else if (lookup.Name == "Array" &&
                             Parser.Settings.IsModificationAllowed(TreeModifications.NewArrayToArrayLiteral))
                    {
                        // Array is trickier.
                        // If there are no arguments, then just use [].
                        // if there are multiple arguments, then use [arg0,arg1...argN].
                        // but if there is one argument and it's numeric, we can't crunch it.
                        // also can't crunch if it's a function call or a member or something, since we won't
                        // KNOW whether or not it's numeric.
                        //
                        // so first see if it even is a single-argument constant wrapper.
                        ConstantWrapper constWrapper = (m_args != null && m_args.Count == 1 ? m_args[0] as ConstantWrapper : null);

                        // if the argument count is not one, then we crunch.
                        // if the argument count IS one, we only crunch if we have a constant wrapper,
                        // AND it's not numeric.
                        if (m_args == null ||
                            m_args.Count != 1 ||
                            (constWrapper != null && !constWrapper.IsNumericLiteral))
                        {
                            // create the new array literal object
                            ArrayLiteral arrayLiteral = new ArrayLiteral(Context, Parser, m_args);
                            // replace ourself within our parent
                            if (Parent.ReplaceChild(this, arrayLiteral))
                            {
                                // recurse
                                arrayLiteral.AnalyzeNode();
                                // and bail -- we don't want to recurse this node any more
                                return;
                            }
                        }
                    }
                }
            }

            // if we are replacing resource references with strings generated from resource files
            // and this is a brackets call: lookup[args]
            ResourceStrings resourceStrings = Parser.ResourceStrings;

            if (m_inBrackets && resourceStrings != null && resourceStrings.Count > 0)
            {
                // see if the root object is a lookup that corresponds to the
                // global value (not a local field) for our resource object
                // (same name)
                Lookup rootLookup = m_func as Lookup;
                if (rootLookup != null &&
                    rootLookup.LocalField == null &&
                    string.CompareOrdinal(rootLookup.Name, resourceStrings.Name) == 0)
                {
                    // we're going to replace this node with a string constant wrapper
                    // but first we need to make sure that this is a valid lookup.
                    // if the parameter contains anything that would vary at run-time,
                    // then we need to throw an error.
                    // the parser will always have either one or zero nodes in the arguments
                    // arg list. We're not interested in zero args, so just make sure there is one
                    if (m_args.Count == 1)
                    {
                        // must be a constant wrapper
                        ConstantWrapper argConstant = m_args[0] as ConstantWrapper;
                        if (argConstant != null)
                        {
                            string resourceName = argConstant.Value.ToString();

                            // get the localized string from the resources object
                            ConstantWrapper resourceLiteral = new ConstantWrapper(
                                resourceStrings[resourceName],
                                PrimitiveType.String,
                                Context,
                                Parser);

                            // replace this node with localized string, analyze it, and bail
                            // so we don't anaylze the tree we just replaced
                            Parent.ReplaceChild(this, resourceLiteral);
                            resourceLiteral.AnalyzeNode();
                            return;
                        }
                        else
                        {
                            // error! must be a constant
                            Context.HandleError(
                                JSError.ResourceReferenceMustBeConstant,
                                true);
                        }
                    }
                    else
                    {
                        // error! can only be a single constant argument to the string resource object.
                        // the parser will only have zero or one arguments, so this must be zero
                        // (since the parser won't pass multiple args to a [] operator)
                        Context.HandleError(
                            JSError.ResourceReferenceMustBeConstant,
                            true);
                    }
                }
            }

            // and finally, if this is a backets call and the argument is a constantwrapper that can
            // be an identifier, just change us to a member node:  obj["prop"] to obj.prop.
            // but ONLY if the string value is "safe" to be an identifier. Even though the ECMA-262
            // spec says certain Unicode categories are okay, in practice the various major browsers
            // all seem to have problems with certain characters in identifiers. Rather than risking
            // some browsers breaking when we change this syntax, don't do it for those "danger" categories.
            if (m_inBrackets && m_args != null)
            {
                // see if there is a single, constant argument
                string argText = m_args.SingleConstantArgument;
                if (argText != null)
                {
                    // see if we want to replace the name
                    string newName;
                    if (Parser.Settings.HasRenamePairs && Parser.Settings.ManualRenamesProperties &&
                        Parser.Settings.IsModificationAllowed(TreeModifications.PropertyRenaming) &&
                        !string.IsNullOrEmpty(newName = Parser.Settings.GetNewName(argText)))
                    {
                        // yes -- we are going to replace the name, either as a string literal, or by converting
                        // to a member-dot operation.
                        // See if we can't turn it into a dot-operator. If we can't, then we just want to replace the operator with
                        // a new constant wrapper. Otherwise we'll just replace the operator with a new constant wrapper.
                        if (Parser.Settings.IsModificationAllowed(TreeModifications.BracketMemberToDotMember) &&
                            JSScanner.IsSafeIdentifier(newName) &&
                            !JSScanner.IsKeyword(newName))
                        {
                            // the new name is safe to convert to a member-dot operator.
                            // but we don't want to convert the node to the NEW name, because we still need to Analyze the
                            // new member node -- and it might convert the new name to something else. So instead we're
                            // just going to convert this existing string to a member node WITH THE OLD STRING,
                            // and THEN analyze it (which will convert the old string to newName)
                            Member replacementMember = new Member(Context, Parser, m_func, argText);
                            Parent.ReplaceChild(this, replacementMember);

                            // this analyze call will convert the old-name member to the newName value
                            replacementMember.AnalyzeNode();
                            return;
                        }
                        else
                        {
                            // nope; can't convert to a dot-operator.
                            // we're just going to replace the first argument with a new string literal
                            // and continue along our merry way.
                            m_args.ReplaceChild(m_args[0], new ConstantWrapper(newName, PrimitiveType.String, m_args[0].Context, Parser));
                        }
                    }
                    else if (Parser.Settings.IsModificationAllowed(TreeModifications.BracketMemberToDotMember) &&
                             JSScanner.IsSafeIdentifier(argText) &&
                             !JSScanner.IsKeyword(argText))
                    {
                        // not a replacement, but the string literal is a safe identifier. So we will
                        // replace this call node with a Member-dot operation
                        Member replacementMember = new Member(Context, Parser, m_func, argText);
                        Parent.ReplaceChild(this, replacementMember);
                        replacementMember.AnalyzeNode();
                        return;
                    }
                }
            }

            // call the base class to recurse
            base.AnalyzeNode();

            // call this AFTER recursing to give the fields a chance to resolve, because we only
            // want to make this replacement if we are working on the global Date object.
            if (!m_inBrackets && !m_isConstructor &&
                (m_args == null || m_args.Count == 0) &&
                member != null && string.CompareOrdinal(member.Name, "getTime") == 0 &&
                Parser.Settings.IsModificationAllowed(TreeModifications.DateGetTimeToUnaryPlus))
            {
                // this is not a constructor and it's not a brackets call, and there are no arguments.
                // if the function is a member operation to "getTime" and the object of the member is a
                // constructor call to the global "Date" object (not a local), then we want to replace the call
                // with a unary plus on the Date constructor. Converting to numeric type is the same as
                // calling getTime, so it's the equivalent with much fewer bytes.
                CallNode dateConstructor = member.Root as CallNode;
                if (dateConstructor != null &&
                    dateConstructor.IsConstructor)
                {
                    // lookup for the predifined (not local) "Date" field
                    Lookup lookup = dateConstructor.Function as Lookup;
                    if (lookup != null && string.CompareOrdinal(lookup.Name, "Date") == 0 &&
                        lookup.LocalField == null)
                    {
                        // this is in the pattern: (new Date()).getTime()
                        // we want to replace it with +new Date
                        // use the same date constructor node as the operand
                        NumericUnary unary = new NumericUnary(Context, Parser, dateConstructor, JSToken.Plus);

                        // replace us (the call to the getTime method) with this unary operator
                        Parent.ReplaceChild(this, unary);

                        // don't need to AnalyzeNode on the unary operator. The operand has already
                        // been analyzed when we recursed, and the unary operator wouldn't do anything
                        // special anyway (since the operand is not a numeric constant)
                    }
                }
            }
            else if (Parser.Settings.EvalTreatment != EvalTreatment.Ignore)
            {
                // if this is a window.eval call, then we need to mark this scope as unknown just as
                // we would if this was a regular eval call.
                // (unless, of course, the parser settings say evals are safe)
                // call AFTER recursing so we know the left-hand side properties have had a chance to
                // lookup their fields to see if they are local or global
                if (member != null && string.CompareOrdinal(member.Name, "eval") == 0)
                {
                    if (member.LeftHandSide.IsWindowLookup)
                    {
                        // this is a call to window.eval()
                        // mark this scope as unknown so we don't crunch out locals
                        // we might reference in the eval at runtime
                        ScopeStack.Peek().IsKnownAtCompileTime = false;
                    }
                }
                else
                {
                    CallNode callNode = m_func as CallNode;
                    if (callNode != null &&
                        callNode.InBrackets &&
                        callNode.LeftHandSide.IsWindowLookup &&
                        callNode.Arguments.IsSingleConstantArgument("eval"))
                    {
                        // this is a call to window["eval"]
                        // mark this scope as unknown so we don't crunch out locals
                        // we might reference in the eval at runtime
                        ScopeStack.Peek().IsKnownAtCompileTime = false;
                    }
                }
            }

            /* REVIEW: may be too late. lookups may alread have been analyzed and
             * found undefined
             * // check to see if this is an assignment to a window["prop"] structure
             * BinaryOperator binaryOp = Parent as BinaryOperator;
             * if (binaryOp != null && binaryOp.IsAssign
             *  && m_inBrackets
             *  && m_func.IsWindowLookup
             *  && m_args != null)
             * {
             *  // and IF the property is a non-empty constant that isn't currently
             *  // a global field...
             *  string propertyName = m_args.SingleConstantArgument;
             *  if (!string.IsNullOrEmpty(propertyName)
             *      && Parser.GlobalScope[propertyName] == null)
             *  {
             *      // we want to also add it to the global fields so it's not undefined
             *      Parser.GlobalScope.DeclareField(propertyName, null, 0);
             *  }
             * }
             */
        }
 private JSScanner GetJSScanner() {
     if (_scanner == null) {
         _scanner = new JSScanner(new CodeSettings() { AllowShebangLine = true });
     }
     return _scanner;
 }
        private LineTokenization TokenizeLine(JSScanner JSScanner, ITextSnapshot snapshot, object previousLineState, int lineNo) {
            ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo);
            SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak);

            var tcp = new SnapshotSpanSourceCodeReader(lineSpan);

            JSScanner.Initialize(
                lineSpan.GetText(),
                previousLineState,
                new SourceLocation(0, lineNo + 1, 1)
            );
            try {
                var tokens = JSScanner.ReadTokens(lineSpan.Length).Select(ToTokenKind).ToArray();
                return new LineTokenization(tokens, JSScanner.CurrentState);
            } finally {
                JSScanner.Uninitialize();
            }
        }
        /// <summary>
        /// Rescans the part of the buffer affected by a change. 
        /// Scans a contiguous sub-<paramref name="span"/> of a larger code span which starts at <paramref name="codeStartLine"/>.
        /// </summary>
        private void ApplyChange(JSScanner JSScanner, ITextSnapshot snapshot, Span span) {
            int firstLine = snapshot.GetLineNumberFromPosition(span.Start);
            int lastLine = snapshot.GetLineNumberFromPosition(span.Length > 0 ? span.End - 1 : span.End);

            Contract.Assert(firstLine >= 0);

            // find the closest line preceding firstLine for which we know categorizer state, stop at the codeStartLine:
            LineTokenization lineTokenization;
            firstLine = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenization) + 1;
            object state = lineTokenization.State;

            int currentLine = firstLine;
            object previousState;
            while (currentLine < snapshot.LineCount) {
                previousState = _tokenCache.TryGetTokenization(currentLine, out lineTokenization) ? lineTokenization.State : null;
                _tokenCache[currentLine] = lineTokenization = TokenizeLine(JSScanner, snapshot, state, currentLine);
                state = lineTokenization.State;

                // stop if we visted all affected lines and the current line has no tokenization state or its previous state is the same as the new state:
                if (currentLine > lastLine && (previousState == null || previousState.Equals(state))) {
                    break;
                }

                currentLine++;
            }

            // classification spans might have changed between the start of the first and end of the last visited line:
            int changeStart = snapshot.GetLineFromLineNumber(firstLine).Start;
            int changeEnd = (currentLine < snapshot.LineCount) ? snapshot.GetLineFromLineNumber(currentLine).End : snapshot.Length;
            if (changeStart < changeEnd) {
                var classificationChanged = ClassificationChanged;
                if (classificationChanged != null) {
                    var args = new ClassificationChangedEventArgs(new SnapshotSpan(snapshot, new Span(changeStart, changeEnd - changeStart)));
                    classificationChanged(this, args);
                }
            }
        }