public bool AddDebugLookup(string debugNamespace) { // a blank identifier is okay -- we just ignore it if (!string.IsNullOrEmpty(debugNamespace)) { // but if it's not blank, it better be a valid JavaScript identifier or member chain if (debugNamespace.IndexOf('.') > 0) { // it's a member chain -- check that each part is a valid JS identifier var names = debugNamespace.Split('.'); foreach (var name in names) { if (!JSScanner.IsValidIdentifier(name)) { return(false); } } } else { // no dot -- just an identifier if (!JSScanner.IsValidIdentifier(debugNamespace)) { return(false); } } m_debugLookups.Add(debugNamespace); return(true); } return(false); }
public override string ToCode(ToCodeFormat format) { StringBuilder sb = new StringBuilder(); sb.Append("do"); ToCodeFormat bodyFormat = ((Body != null && Body.Count == 1 && Body[0].GetType() == typeof(DoWhile)) ? ToCodeFormat.AlwaysBraces : ToCodeFormat.Normal ); // if the body is a single statement that ends in a do-while, then we // will need to wrap the body in curly-braces to get around an IE bug if (Body != null && Body.EncloseBlock(EncloseBlockType.SingleDoWhile)) { bodyFormat = ToCodeFormat.AlwaysBraces; } string bodyString = ( Body == null ? string.Empty : Body.ToCode(bodyFormat) ); if (bodyString.Length == 0) { sb.Append(';'); } else { // if the first character could be interpreted as a continuation // of the "do" keyword, then we need to add a space if (JSScanner.StartsWithIdentifierPart(bodyString)) { sb.Append(' '); } sb.Append(bodyString); // if there is no body, we need a semi-colon // OR if we didn't always wrap in braces AND we require a separator, we need a semi-colon. // and make sure it doesn't already end in a semicolon -- we don't want two in a row. if (Body == null || (bodyFormat != ToCodeFormat.AlwaysBraces && Body.RequiresSeparator && !bodyString.EndsWith(";", StringComparison.Ordinal))) { sb.Append(';'); } } // add a space for readability for pretty-print mode if (Parser.Settings.OutputMode == OutputMode.MultipleLines && Parser.Settings.IndentSize > 0) { sb.Append(' '); } sb.Append("while("); sb.Append(Condition.ToCode()); sb.Append(")"); return(sb.ToString()); }
/// <summary> /// Add a known global identifier to the list /// </summary> /// <param name="identifier">global identifier</param> /// <returns>true if valid identifier; false if invalid identifier</returns> public bool AddKnownGlobal(string identifier) { if (JSScanner.IsValidIdentifier(identifier)) { m_knownGlobals.Add(identifier); return(true); } return(false); }
public bool AddNoAutoRename(string noRename) { if (!JSScanner.IsValidIdentifier(noRename)) { return(false); } m_noRenameSet.Add(noRename); return(true); }
private JSScanner GetJSScanner() { if (_scanner == null) { _scanner = new JSScanner(new CodeSettings() { AllowShebangLine = true }); } return(_scanner); }
private static List <TokenWithSpan> ReadTokens(string code, bool collectWarnings, params ErrorInfo[] errors) { CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings); var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true }); var tokens = scanner.ReadTokens(Int32.MaxValue); errorSink.CheckErrors(errors); return(tokens); }
internal string NextName() { string name; do { // advance to the next name ++m_currentName; name = CurrentName; // keep advancing until we find one that isn't in the skip list or a keyword }while (m_skipNames.ContainsKey(name) || JSScanner.IsKeyword(name)); return(name); }
// Determine if an identifier is valid for this engine. public override bool IsValidIdentifier(String identifier) { if (identifier == null) { return(false); } JSScanner scanner = new JSScanner(identifier); if (scanner.FetchIdentifier() == null) { return(false); } return(scanner.Fetch() == -1); }
public override string ToCode(ToCodeFormat format) { StringBuilder sb = new StringBuilder(); sb.Append("return"); if (m_operand != null) { string operandText = m_operand.ToCode(); if (operandText.Length > 0 && JSScanner.StartsWithIdentifierPart(operandText)) { sb.Append(' '); } sb.Append(operandText); } return(sb.ToString()); }
public override string ToCode(ToCodeFormat format) { if (PrimitiveType == PrimitiveType.String) { string rawValue = Value.ToString(); // if the raw value is safe to be an identifier, then go ahead and ditch the quotes and just output // the raw value. Otherwise call ToCode to wrap the string in quotes. return(JSScanner.IsSafeIdentifier(rawValue) && !JSScanner.IsKeyword(rawValue) ? rawValue : base.ToCode(format)); } else { // call the base to format the value return(base.ToCode(format)); } }
/// <summary> /// Set the dictionary of preprocessor defines and values /// </summary> /// <param name="defines">dictionary to set</param> public int SetPreprocessorValues(IDictionary <string, string> defines) { PreprocessorValues.Clear(); if (defines != null && defines.Count > 0) { foreach (var define in defines) { if (JSScanner.IsValidIdentifier(define.Key)) { PreprocessorValues.Add(define.Key, define.Value); } } } return(PreprocessorValues.Count); }
private static bool NeedsParens(AstNode node, JSToken refToken) { bool needsParens = false; // assignments and commas are the only operators that need parens // around them. Conditional is pretty low down the list BinaryOperator binaryOp = node as BinaryOperator; if (binaryOp != null) { OpPrec thisPrecedence = JSScanner.GetOperatorPrecedence(refToken); OpPrec nodePrecedence = JSScanner.GetOperatorPrecedence(binaryOp.OperatorToken); needsParens = (nodePrecedence < thisPrecedence); } return(needsParens); }
/// <summary> /// Rescans the part of the buffer affected by a change. /// Scans a contiguous sub-<paramref name="span"/> of a larger code span which starts at <paramref name="codeStartLine"/>. /// </summary> private void ApplyChange(JSScanner JSScanner, ITextSnapshot snapshot, Span span) { int firstLine = snapshot.GetLineNumberFromPosition(span.Start); int lastLine = snapshot.GetLineNumberFromPosition(span.Length > 0 ? span.End - 1 : span.End); Contract.Assert(firstLine >= 0); // find the closest line preceding firstLine for which we know categorizer state, stop at the codeStartLine: LineTokenization lineTokenization; firstLine = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenization) + 1; object state = lineTokenization.State; int currentLine = firstLine; object previousState; while (currentLine < snapshot.LineCount) { previousState = _tokenCache.TryGetTokenization(currentLine, out lineTokenization) ? lineTokenization.State : null; _tokenCache[currentLine] = lineTokenization = TokenizeLine(JSScanner, snapshot, state, currentLine); state = lineTokenization.State; // stop if we visted all affected lines and the current line has no tokenization state or its previous state is the same as the new state: if (currentLine > lastLine && (previousState == null || previousState.Equals(state))) { break; } currentLine++; } // classification spans might have changed between the start of the first and end of the last visited line: int changeStart = snapshot.GetLineFromLineNumber(firstLine).Start; int changeEnd = (currentLine < snapshot.LineCount) ? snapshot.GetLineFromLineNumber(currentLine).End : snapshot.Length; if (changeStart < changeEnd) { var classificationChanged = ClassificationChanged; if (classificationChanged != null) { var args = new ClassificationChangedEventArgs(new SnapshotSpan(snapshot, new Span(changeStart, changeEnd - changeStart))); classificationChanged(this, args); } } }
private static List <TokenWithSpan> ScanTokens(string code, bool collectWarnings, params ErrorInfo[] errors) { CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings); var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true }); List <TokenWithSpan> tokens = new List <TokenWithSpan>(); for (TokenWithSpan curToken = scanner.ScanNextTokenWithSpan(true); curToken.Token != JSToken.EndOfFile; curToken = scanner.ScanNextTokenWithSpan(true)) { tokens.Add(curToken); } errorSink.CheckErrors(errors); return(tokens); }
// Test a single operator. private void TestOp(JSScanner scanner, String ops, ref int posn, JSToken expected) { Context token = JSScannerTest.TestGetTokenContext(scanner); scanner.GetNextToken(); int next = ops.IndexOf(' ', posn); String thisop = ops.Substring(posn, next - posn); AssertEquals("TestOp[" + thisop + "] (1)", expected, token.GetToken()); AssertEquals("TestOp[" + thisop + "] (2)", thisop, token.GetCode()); AssertEquals("TestOp[" + thisop + "] (3)", posn, token.StartPosition); AssertEquals("TestOp[" + thisop + "] (4)", next, token.EndPosition); posn = next + 1; }
private LineTokenization TokenizeLine(JSScanner JSScanner, ITextSnapshot snapshot, object previousLineState, int lineNo) { ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo); SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak); var tcp = new SnapshotSpanSourceCodeReader(lineSpan); JSScanner.Initialize( lineSpan.GetText(), previousLineState, new SourceLocation(0, lineNo + 1, 1) ); try { var tokens = JSScanner.ReadTokens(lineSpan.Length).Select(ToTokenKind).ToArray(); return(new LineTokenization(tokens, JSScanner.CurrentState)); } finally { JSScanner.Uninitialize(); } }
public static string CrunchedLabel(int nestLevel) { // nestCount is 0-based. // return null if the nestLevel is invalid (< 0). string minLabel = null; if (nestLevel >= 0) { minLabel = GenerateNameFromNumber(nestLevel); if (JSScanner.IsKeyword(minLabel, true)) { // prepend something to make it NOT a keyword. // no keywords start with an underscore, so... minLabel = '_' + minLabel; } } return(minLabel); }
private static bool IsOnlyDecimalDigits(string text) { // if text is null, return false. // Otherwise return true if ALL the characters are decimal digits, // or false is ANY ONE character isn't. //return text.IfNotNull(s => !s.Any(c => !JSScanner.IsDigit(c))); return(text.IfNotNull(s => { foreach (char c in s) { if (!JSScanner.IsDigit(c)) { return false; } } return true; })); }
public override string ToCode(ToCodeFormat format) { string operandString = Operand.ToCode(format); if (NeedsParentheses) { // needs parens return("delete(" + operandString + ')'); } else if (JSScanner.StartsWithIdentifierPart(operandString)) { // needs a space return("delete " + operandString); } else { // needs no separator return("delete" + operandString); } }
public override string ToCode(ToCodeFormat format) { string operandString = Operand.ToCode(format); if (NeedsParentheses) { // need parentheses return("typeof(" + operandString + ')'); } else if (JSScanner.StartsWithIdentifierPart(operandString)) { // need a space separating them return("typeof " + operandString); } else { // don't need the space return("typeof" + operandString); } }
public override string ToCode(ToCodeFormat format) { StringBuilder sb = new StringBuilder(); // the label should be indented Parser.Settings.Indent(); // start a new line Parser.Settings.NewLine(sb); if (m_caseValue != null) { sb.Append("case"); string caseValue = m_caseValue.ToCode(); if (JSScanner.StartsWithIdentifierPart(caseValue)) { sb.Append(' '); } sb.Append(caseValue); } else { sb.Append("default"); } sb.Append(':'); // in pretty-print mode, we indent the statements under the label, too Parser.Settings.Indent(); // output the statements if (m_statements != null && m_statements.Count > 0) { sb.Append(m_statements.ToCode(ToCodeFormat.NoBraces)); } // if we are pretty-printing, we need to unindent twice: // once for the label, and again for the statements Parser.Settings.Unindent(); Parser.Settings.Unindent(); return(sb.ToString()); }
private void UnreferencedFunction(JSVariableField variableField, FunctionObject functionObject) { // if there is no name, then ignore this declaration because it's malformed. // (won't be a function expression because those are automatically referenced). // also ignore ghosted function fields. if (functionObject.Name != null && variableField.FieldType != FieldType.GhostFunction) { // if the function name isn't a simple identifier, then leave it there and mark it as // not renamable because it's probably one of those darn IE-extension event handlers or something. if (JSScanner.IsValidIdentifier(functionObject.Name)) { // unreferenced function declaration. fire a warning. var ctx = functionObject.IdContext ?? variableField.OriginalContext; ctx.HandleError(JSError.FunctionNotReferenced, false); // hide it from the output if our settings say we can. // we don't want to delete it, per se, because we still want it to // show up in the scope report so the user can see that it was unreachable // in case they are wondering where it went. // ES6 has the notion of block-scoped function declarations. ES5 says functions can't // be defined inside blocks -- only at the root level of the global scope or function scopes. // so if this is a block scope, don't hide the function, even if it is unreferenced because // of the cross-browser difference. if (this.IsKnownAtCompileTime && m_settings.MinifyCode && m_settings.RemoveUnneededCode && !(this is BlockScope)) { functionObject.HideFromOutput = true; } } else { // not a valid identifier name for this function. Don't rename it because it's // malformed and we don't want to mess up the developer's intent. variableField.CanCrunch = false; } } }
// Determine if a namespace name is valid. protected override bool IsValidNamespaceName(String name) { if (name == null) { return(false); } JSScanner scanner = new JSScanner(name); if (scanner.FetchIdentifier() == null) { return(false); } while (scanner.Peek() == '.') { scanner.Fetch(); if (scanner.FetchIdentifier() == null) { return(false); } } return(scanner.Fetch() == -1); }
public void PartialScanning() { var code1 = "/* hello world "; var code2 = " goodbye */"; CollectingErrorSink errorSink = new CollectingErrorSink(true); var scanner = new JSScanner(code1, errorSink, new CodeSettings() { AllowShebangLine = true }); var tokens = scanner.ReadTokens(Int32.MaxValue); VerifyTokens( tokens, new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(0, 1, 1), new SourceLocation(15, 1, 16)) ); scanner.Initialize(code2, scanner.CurrentState, new SourceLocation(code1.Length, 2, 1)); tokens = scanner.ReadTokens(Int32.MaxValue); VerifyTokens( tokens, new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(15, 2, 16), new SourceLocation(28, 2, 29)) ); Assert.IsTrue(scanner.CurrentState.Equals(scanner.CurrentState)); Assert.IsFalse(scanner.CurrentState.Equals(new object())); Assert.AreEqual(scanner.CurrentState.GetHashCode(), 2); }
/// <summary> /// We really only care about caching strings that are likely /// to be identifiers. Everything else we'll coalesce into the /// empty string. /// </summary> private static bool DontCacheString(string strValue) { if (strValue.Length > 100) { return(true); } for (int i = 0; i < strValue.Length; i++) { char ch = strValue[i]; if (ch == '-' || ch == '.' || ch == '/' || ch == '\\') { // not valid identifiers, but likely to show up in // require string literals, so we still care about preserving // these strings. continue; } if (!JSScanner.IsValidIdentifierPart(strValue[i])) { return(true); } } return(false); }
public bool Match(AstNode node, string identifiers) { // set the match to false m_isMatch = false; // identifiers cannot be null or blank and must match: IDENT(.IDENT)* // since for JS there has to be at least a global object, the dot must be AFTER the first character. if (node != null && !string.IsNullOrEmpty(identifiers)) { // get all the parts var parts = identifiers.Split('.'); // each part must be a valid JavaScript identifier. Assume everything is valid // unless at least one is invalid -- then forget it var isValid = true; foreach (var part in parts) { if (!JSScanner.IsValidIdentifier(part)) { isValid = false; break; } } // must be valid to continue if (isValid) { // save the parts and start the index on the last one, since we'll be walking backwards m_parts = parts; m_index = parts.Length - 1; node.Accept(this); } } return(m_isMatch); }
/// <summary> /// Add a rename pair to the identifier rename map /// </summary> /// <param name="sourceName">name of the identifier in the source code</param> /// <param name="newName">new name with which to replace the source name</param> /// <returns>true if added; false if either name is not a valid JavaScript identifier</returns> public bool AddRenamePair(string sourceName, string newName) { bool successfullyAdded = false; // both names MUST be valid JavaScript identifiers if (JSScanner.IsValidIdentifier(sourceName) && JSScanner.IsValidIdentifier(newName)) { if (m_identifierReplacementMap.ContainsKey(sourceName)) { // just replace the value m_identifierReplacementMap[sourceName] = newName; } else { // add the new pair m_identifierReplacementMap.Add(sourceName, newName); } // if we get here, we added it (or updated it if it's a dupe) successfullyAdded = true; } return(successfullyAdded); }
private LineTokenization GetPreviousTokenization(JSScanner JSScanner, ITextSnapshot snapshot, int firstLine, int prevLine) { LineTokenization prevLineTokenization; if (!_tokenCache.TryGetTokenization(prevLine, out prevLineTokenization)) { LineTokenization lineTokenizationTemp; int currentLineTemp = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenizationTemp) + 1; object stateTemp = lineTokenizationTemp.State; while (currentLineTemp < snapshot.LineCount) { if (!_tokenCache.TryGetTokenization(currentLineTemp, out lineTokenizationTemp)) { lineTokenizationTemp = TokenizeLine(JSScanner, snapshot, stateTemp, currentLineTemp); _tokenCache[currentLineTemp] = lineTokenizationTemp; } currentLineTemp++; stateTemp = lineTokenizationTemp.State; } prevLineTokenization = TokenizeLine(JSScanner, snapshot, stateTemp, prevLine); _tokenCache[prevLine] = prevLineTokenization; } return prevLineTokenization; }
static string CreateJSFromResourceStrings(ResourceStrings resourceStrings) { var sb = StringBuilderPool.Acquire(); try { // start the var statement using the requested name and open the initializer object literal sb.Append("var "); sb.Append(resourceStrings.Name); sb.Append("={"); // we're going to need to insert commas between each pair, so we'll use a boolean // flag to indicate that we're on the first pair. When we output the first pair, we'll // set the flag to false. When the flag is false, we're about to insert another pair, so // we'll add the comma just before. bool firstItem = true; // loop through all items in the collection foreach (var keyPair in resourceStrings.NameValuePairs) { // if this isn't the first item, we need to add a comma separator if (!firstItem) { sb.Append(','); } else { // next loop is no longer the first item firstItem = false; } // append the key as the name, a colon to separate the name and value, // and then the value // must quote if not valid JS identifier format, or if it is, but it's a keyword // (use strict mode just to be safe) string propertyName = keyPair.Key; if (!JSScanner.IsValidIdentifier(propertyName) || JSScanner.IsKeyword(propertyName, true)) { sb.Append("\""); // because we are using quotes for the delimiters, replace any instances // of a quote character (") with an escaped quote character (\") sb.Append(propertyName.Replace("\"", "\\\"")); sb.Append("\""); } else { sb.Append(propertyName); } sb.Append(':'); // make sure the Value is properly escaped, quoted, and whatever we // need to do to make sure it's a proper JS string. // pass false for whether this string is an argument to a RegExp constructor. // pass false for whether to use W3Strict formatting for character escapes (use maximum browser compatibility) // pass true for ecma strict mode string stringValue = ConstantWrapper.EscapeString( keyPair.Value, false, false, true ); sb.Append(stringValue); } // close the object literal and return the string sb.AppendLine("};"); return(sb.ToString()); } finally { sb.Release(); } }
/// <summary> /// Creates an instance of the JSParser class that can be used to parse the given source code. /// </summary> /// <param name="source">Source code to parse.</param> public JSParser(string source) { m_severity = 5; m_blockType = new List<BlockType>(16); m_labelTable = new Dictionary<string, LabelInfo>(); m_noSkipTokenSet = new NoSkipTokenSet(); m_importantComments = new List<Context>(); m_document = new DocumentContext(this, source); m_scanner = new JSScanner(new Context(m_document)); m_currentToken = new Context(m_document); // if the scanner encounters a special "globals" comment, it'll fire this event // at which point we will define a field with that name in the global scope. m_scanner.GlobalDefine += (sender, ea) => { var globalScope = GlobalScope; if (globalScope[ea.Name] == null) { var field = globalScope.CreateField(ea.Name, null, FieldAttributes.SpecialName); globalScope.AddField(field); } }; // this event is fired whenever a ///#SOURCE comment is encountered m_scanner.NewModule += (sender, ea) => { m_newModule = true; }; }
// Test a single operator. private void TestOp(JSScanner scanner, String ops, ref int posn, JSToken expected) { Context token = JSScannerTest.TestGetTokenContext(scanner); scanner.GetNextToken(); int next = ops.IndexOf(' ', posn); String thisop = ops.Substring(posn, next - posn); AssertEquals("TestOp[" + thisop + "] (1)", expected, token.GetToken()); AssertEquals("TestOp[" + thisop + "] (2)", thisop, token.GetCode()); AssertEquals("TestOp[" + thisop + "] (3)", posn, token.StartPosition); AssertEquals("TestOp[" + thisop + "] (4)", next, token.EndPosition); posn = next + 1; }
private static List<TokenWithSpan> ScanTokens(string code, bool collectWarnings, params ErrorInfo[] errors) { CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings); var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true }); List<TokenWithSpan> tokens = new List<TokenWithSpan>(); for (TokenWithSpan curToken = scanner.ScanNextTokenWithSpan(true); curToken.Token != JSToken.EndOfFile; curToken = scanner.ScanNextTokenWithSpan(true)) { tokens.Add(curToken); } errorSink.CheckErrors(errors); return tokens; }
private static List<TokenWithSpan> ReadTokens(string code, bool collectWarnings, params ErrorInfo[] errors) { CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings); var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true }); var tokens = scanner.ReadTokens(Int32.MaxValue); errorSink.CheckErrors(errors); return tokens; }
internal override Dictionary <string, IAnalysisSet> GetOwnProperties(ProjectEntry accessor) { if (_descriptors == null || _descriptors.Count == 0) { return(new Dictionary <string, IAnalysisSet>()); } var res = new Dictionary <string, IAnalysisSet>(); foreach (var kvp in _descriptors) { var key = kvp.Key; if (!JSScanner.IsValidIdentifier(key)) { // https://nodejstools.codeplex.com/workitem/987 // for intellisense purposes we don't report invalid identifiers, but we can // end up with these from indexing with constant strings. continue; } if (kvp.Value.Values != null) { //kvp.Value.Values.ClearOldValues(); if (kvp.Value.Values.VariableStillExists) { var types = kvp.Value.Values.GetTypesNoCopy(accessor, ProjectEntry); if (types.Count != 0 || kvp.Value.Values.TypesNoCopy.Count == 0) { MergeTypes(res, key, types); } } } if (kvp.Value.Getter != null) { foreach (var value in kvp.Value.Getter.GetTypesNoCopy(accessor, ProjectEntry)) { FunctionValue userFunc = value.Value as FunctionValue; if (userFunc != null) { MergeTypes(res, key, userFunc.ReturnTypes); } } } if (kvp.Value.Setter != null) { MergeTypes(res, key, AnalysisSet.Empty); } } if (_linkedValues != null) { foreach (var linkedValue in _linkedValues.GetTypesNoCopy(accessor, ProjectEntry)) { if (linkedValue.Value.Push()) { try { MergeDictionaries(res, linkedValue.Value.GetAllMembers(accessor)); } finally { linkedValue.Value.Pop(); } } } } return(res); }
/// <summary> /// Adds classification spans to the given collection. /// Scans a contiguous sub-<paramref name="span"/> of a larger code span which starts at <paramref name="codeStartLine"/>. /// </summary> private void AddClassifications(JSScanner JSScanner, List<ClassificationSpan> classifications, SnapshotSpan span) { Debug.Assert(span.Length > 0); var snapshot = span.Snapshot; int firstLine = snapshot.GetLineNumberFromPosition(span.Start); int lastLine = snapshot.GetLineNumberFromPosition(span.End - 1); Contract.Assert(firstLine >= 0); _tokenCache.EnsureCapacity(snapshot.LineCount); // find the closest line preceding firstLine for which we know categorizer state, stop at the codeStartLine: LineTokenization lineTokenization; int currentLine = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenization) + 1; object state = lineTokenization.State; // track the previous 2 tokens to adjust our classifications of keywords // when they shouldn't be displayed as keywords... TokenInfoWithLine? prevToken = null, prevPrevToken = null; // initialize the previous tokens so we can handle things like: // foo. // get() // even if we're called on the line for get() int prevLine = currentLine - 1; while (prevLine >= 0 && prevToken == null) { LineTokenization prevLineTokenization = GetPreviousTokenization(JSScanner, snapshot, firstLine, prevLine); for (int i = prevLineTokenization.Tokens.Length - 1; i >= 0 && prevToken == null; i--) { var tempToken = prevLineTokenization.Tokens[i]; if (IsValidPreviousToken(ref tempToken)) { prevToken = prevPrevToken; prevPrevToken = new TokenInfoWithLine() { TokenInfo = tempToken, Line = prevLine }; } } prevLine--; } while (currentLine <= lastLine) { if (!_tokenCache.TryGetTokenization(currentLine, out lineTokenization)) { lineTokenization = TokenizeLine(JSScanner, snapshot, state, currentLine); _tokenCache[currentLine] = lineTokenization; } state = lineTokenization.State; for (int i = 0; i < lineTokenization.Tokens.Length; i++) { var token = lineTokenization.Tokens[i]; if (token.Category == TokenCategory.IncompleteMultiLineStringLiteral || token.Category == TokenCategory.Comment) { IClassificationType type; switch (token.Category) { case TokenCategory.IncompleteMultiLineStringLiteral: type = _provider.StringLiteral; break; case TokenCategory.Comment: type = _provider.Comment; break; default: type = null; break; } Debug.Assert(type != null, "We should have a defined ClassificationType for every token."); // we need to walk backwards to find the start of this multi-line string... TokenInfo startToken = token; int validPrevLine; int length = startToken.SourceSpan.Length; if (i == 0) { length += GetLeadingMultiLineTokens(JSScanner, snapshot, token.Category, firstLine, currentLine, out validPrevLine, ref startToken); } else { validPrevLine = currentLine; } if (i == lineTokenization.Tokens.Length - 1) { length += GetTrailingMultiLineTokens(JSScanner, snapshot, token.Category, currentLine, state); } var tokenSpan = new Span(SnapshotSpanToSpan(snapshot, startToken, validPrevLine).Start, length); var intersection = span.Intersection(tokenSpan); if ((intersection != null && intersection.Value.Length > 0) || (span.Length == 0 && tokenSpan.Contains(span.Start)) // handle zero-length spans ) { classifications.Add(new ClassificationSpan(new SnapshotSpan(snapshot, tokenSpan), type)); } } else { ClassificationSpan classification = null; if (token.Category == TokenCategory.Keyword) { // check and see if we're not really a keyword... if (IsKeywordInIdentifierContext(snapshot, prevToken, prevPrevToken, new TokenInfoWithLine() { TokenInfo = token, Line = currentLine })) { classification = GetClassificationSpan( span, token, currentLine, CategoryMap[TokenCategory.Identifier] ); } } if (classification == null) { classification = ClassifyToken(span, token, currentLine); } if (classification != null) { classifications.Add(classification); } } if (IsValidPreviousToken(ref token)) { prevPrevToken = prevToken; prevToken = new TokenInfoWithLine() { TokenInfo = token, Line = currentLine }; } } currentLine++; } }
private int GetLeadingMultiLineTokens(JSScanner JSScanner, ITextSnapshot snapshot, TokenCategory tokenCategory, int firstLine, int currentLine, out int validPrevLine, ref TokenInfo startToken) { validPrevLine = currentLine; int prevLine = currentLine - 1; int length = 0; while (prevLine >= 0) { LineTokenization prevLineTokenization = GetPreviousTokenization(JSScanner, snapshot, firstLine, prevLine); if (prevLineTokenization.Tokens.Length != 0) { if (prevLineTokenization.Tokens[prevLineTokenization.Tokens.Length - 1].Category != tokenCategory) { break; } startToken = prevLineTokenization.Tokens[prevLineTokenization.Tokens.Length - 1]; length += startToken.SourceSpan.Length; } validPrevLine = prevLine; prevLine--; if (prevLineTokenization.Tokens.Length > 1) { // http://pytools.codeplex.com/workitem/749 // if there are multiple tokens on this line then our multi-line string // is terminated. break; } } return length; }
public override string ToCode(ToCodeFormat format) { StringBuilder sb = new StringBuilder(); sb.Append("if("); sb.Append(Condition.ToCode()); sb.Append(')'); // if we're in Safari-quirks mode, we will need to wrap the if block // in curly braces if it only includes a function declaration. Safari // throws parsing errors in those situations ToCodeFormat elseFormat = ToCodeFormat.Normal; if (FalseBlock != null && FalseBlock.Count == 1) { if (Parser.Settings.MacSafariQuirks && FalseBlock[0] is FunctionObject) { elseFormat = ToCodeFormat.AlwaysBraces; } else if (FalseBlock[0] is IfNode) { elseFormat = ToCodeFormat.ElseIf; } } // get the else block -- we need to know if there is anything in order // to fully determine if the true-branch needs curly-braces string elseBlock = ( FalseBlock == null ? string.Empty : FalseBlock.ToCode(elseFormat)); // we'll need to force the true block to be enclosed in curly braces if // there is an else block and the true block contains a single statement // that ends in an if that doesn't have an else block ToCodeFormat trueFormat = (FalseBlock != null && TrueBlock != null && TrueBlock.EncloseBlock(EncloseBlockType.IfWithoutElse) ? ToCodeFormat.AlwaysBraces : ToCodeFormat.Normal); if (elseBlock.Length > 0 && TrueBlock != null && TrueBlock.EncloseBlock(EncloseBlockType.SingleDoWhile)) { trueFormat = ToCodeFormat.AlwaysBraces; } // if we're in Safari-quirks mode, we will need to wrap the if block // in curly braces if it only includes a function declaration. Safari // throws parsing errors in those situations if (Parser.Settings.MacSafariQuirks && TrueBlock != null && TrueBlock.Count == 1 && TrueBlock[0] is FunctionObject) { trueFormat = ToCodeFormat.AlwaysBraces; } // add the true block string trueBlock = ( TrueBlock == null ? string.Empty : TrueBlock.ToCode(trueFormat)); sb.Append(trueBlock); if (elseBlock.Length > 0) { if (trueFormat != ToCodeFormat.AlwaysBraces && !trueBlock.EndsWith(";", StringComparison.Ordinal) && (TrueBlock == null || TrueBlock.RequiresSeparator)) { sb.Append(';'); } // if we are in pretty-print mode, drop the else onto a new line Parser.Settings.NewLine(sb); sb.Append("else"); // if the first character could be interpreted as a continuation // of the "else" statement, then we need to add a space if (JSScanner.StartsWithIdentifierPart(elseBlock)) { sb.Append(' '); } sb.Append(elseBlock); } return(sb.ToString()); }
private int GetTrailingMultiLineTokens(JSScanner JSScanner, ITextSnapshot snapshot, TokenCategory tokenCategory, int currentLine, object state) { int nextLine = currentLine + 1; var prevState = state; int length = 0; while (nextLine < snapshot.LineCount) { LineTokenization nextLineTokenization; if (!_tokenCache.TryGetTokenization(nextLine, out nextLineTokenization)) { nextLineTokenization = TokenizeLine(JSScanner, snapshot, prevState, nextLine); prevState = nextLineTokenization.State; _tokenCache[nextLine] = nextLineTokenization; } if (nextLineTokenization.Tokens.Length != 0) { if (nextLineTokenization.Tokens[0].Category != tokenCategory) { break; } length += nextLineTokenization.Tokens[0].SourceSpan.Length; } nextLine++; } return length; }
internal override void AnalyzeNode() { // see if this is a member (we'll need it for a couple checks) Member member = m_func as Member; if (Parser.Settings.StripDebugStatements && Parser.Settings.IsModificationAllowed(TreeModifications.StripDebugStatements)) { // if this is a member, and it's a debugger object, and it's a constructor.... if (member != null && member.IsDebuggerStatement && m_isConstructor) { // we need to replace our debugger object with a generic Object m_func = new Lookup("Object", m_func.Context, Parser); // and make sure the node list is empty if (m_args != null && m_args.Count > 0) { m_args = new AstNodeList(m_args.Context, Parser); } } } // if this is a constructor and we want to collapse // some of them to literals... if (m_isConstructor && Parser.Settings.CollapseToLiteral) { // see if this is a lookup, and if so, if it's pointing to one // of the two constructors we want to collapse Lookup lookup = m_func as Lookup; if (lookup != null) { if (lookup.Name == "Object" && Parser.Settings.IsModificationAllowed(TreeModifications.NewObjectToObjectLiteral)) { // no arguments -- the Object constructor with no arguments is the exact same as an empty // object literal if (m_args == null || m_args.Count == 0) { // replace our node with an object literal ObjectLiteral objLiteral = new ObjectLiteral(Context, Parser, null, null); if (Parent.ReplaceChild(this, objLiteral)) { // and bail now. No need to recurse -- it's an empty literal return; } } else if (m_args.Count == 1) { // one argument // check to see if it's an object literal. ObjectLiteral objectLiteral = m_args[0] as ObjectLiteral; if (objectLiteral != null) { // the Object constructor with an argument that is a JavaScript object merely returns the // argument. Since the argument is an object literal, it is by definition a JavaScript object // and therefore we can replace the constructor call with the object literal Parent.ReplaceChild(this, objectLiteral); // don't forget to recurse the object now objectLiteral.AnalyzeNode(); // and then bail -- we don't want to process this call // operation any more; we've gotten rid of it return; } } } else if (lookup.Name == "Array" && Parser.Settings.IsModificationAllowed(TreeModifications.NewArrayToArrayLiteral)) { // Array is trickier. // If there are no arguments, then just use []. // if there are multiple arguments, then use [arg0,arg1...argN]. // but if there is one argument and it's numeric, we can't crunch it. // also can't crunch if it's a function call or a member or something, since we won't // KNOW whether or not it's numeric. // // so first see if it even is a single-argument constant wrapper. ConstantWrapper constWrapper = (m_args != null && m_args.Count == 1 ? m_args[0] as ConstantWrapper : null); // if the argument count is not one, then we crunch. // if the argument count IS one, we only crunch if we have a constant wrapper, // AND it's not numeric. if (m_args == null || m_args.Count != 1 || (constWrapper != null && !constWrapper.IsNumericLiteral)) { // create the new array literal object ArrayLiteral arrayLiteral = new ArrayLiteral(Context, Parser, m_args); // replace ourself within our parent if (Parent.ReplaceChild(this, arrayLiteral)) { // recurse arrayLiteral.AnalyzeNode(); // and bail -- we don't want to recurse this node any more return; } } } } } // if we are replacing resource references with strings generated from resource files // and this is a brackets call: lookup[args] ResourceStrings resourceStrings = Parser.ResourceStrings; if (m_inBrackets && resourceStrings != null && resourceStrings.Count > 0) { // see if the root object is a lookup that corresponds to the // global value (not a local field) for our resource object // (same name) Lookup rootLookup = m_func as Lookup; if (rootLookup != null && rootLookup.LocalField == null && string.CompareOrdinal(rootLookup.Name, resourceStrings.Name) == 0) { // we're going to replace this node with a string constant wrapper // but first we need to make sure that this is a valid lookup. // if the parameter contains anything that would vary at run-time, // then we need to throw an error. // the parser will always have either one or zero nodes in the arguments // arg list. We're not interested in zero args, so just make sure there is one if (m_args.Count == 1) { // must be a constant wrapper ConstantWrapper argConstant = m_args[0] as ConstantWrapper; if (argConstant != null) { string resourceName = argConstant.Value.ToString(); // get the localized string from the resources object ConstantWrapper resourceLiteral = new ConstantWrapper( resourceStrings[resourceName], PrimitiveType.String, Context, Parser); // replace this node with localized string, analyze it, and bail // so we don't anaylze the tree we just replaced Parent.ReplaceChild(this, resourceLiteral); resourceLiteral.AnalyzeNode(); return; } else { // error! must be a constant Context.HandleError( JSError.ResourceReferenceMustBeConstant, true); } } else { // error! can only be a single constant argument to the string resource object. // the parser will only have zero or one arguments, so this must be zero // (since the parser won't pass multiple args to a [] operator) Context.HandleError( JSError.ResourceReferenceMustBeConstant, true); } } } // and finally, if this is a backets call and the argument is a constantwrapper that can // be an identifier, just change us to a member node: obj["prop"] to obj.prop. // but ONLY if the string value is "safe" to be an identifier. Even though the ECMA-262 // spec says certain Unicode categories are okay, in practice the various major browsers // all seem to have problems with certain characters in identifiers. Rather than risking // some browsers breaking when we change this syntax, don't do it for those "danger" categories. if (m_inBrackets && m_args != null) { // see if there is a single, constant argument string argText = m_args.SingleConstantArgument; if (argText != null) { // see if we want to replace the name string newName; if (Parser.Settings.HasRenamePairs && Parser.Settings.ManualRenamesProperties && Parser.Settings.IsModificationAllowed(TreeModifications.PropertyRenaming) && !string.IsNullOrEmpty(newName = Parser.Settings.GetNewName(argText))) { // yes -- we are going to replace the name, either as a string literal, or by converting // to a member-dot operation. // See if we can't turn it into a dot-operator. If we can't, then we just want to replace the operator with // a new constant wrapper. Otherwise we'll just replace the operator with a new constant wrapper. if (Parser.Settings.IsModificationAllowed(TreeModifications.BracketMemberToDotMember) && JSScanner.IsSafeIdentifier(newName) && !JSScanner.IsKeyword(newName)) { // the new name is safe to convert to a member-dot operator. // but we don't want to convert the node to the NEW name, because we still need to Analyze the // new member node -- and it might convert the new name to something else. So instead we're // just going to convert this existing string to a member node WITH THE OLD STRING, // and THEN analyze it (which will convert the old string to newName) Member replacementMember = new Member(Context, Parser, m_func, argText); Parent.ReplaceChild(this, replacementMember); // this analyze call will convert the old-name member to the newName value replacementMember.AnalyzeNode(); return; } else { // nope; can't convert to a dot-operator. // we're just going to replace the first argument with a new string literal // and continue along our merry way. m_args.ReplaceChild(m_args[0], new ConstantWrapper(newName, PrimitiveType.String, m_args[0].Context, Parser)); } } else if (Parser.Settings.IsModificationAllowed(TreeModifications.BracketMemberToDotMember) && JSScanner.IsSafeIdentifier(argText) && !JSScanner.IsKeyword(argText)) { // not a replacement, but the string literal is a safe identifier. So we will // replace this call node with a Member-dot operation Member replacementMember = new Member(Context, Parser, m_func, argText); Parent.ReplaceChild(this, replacementMember); replacementMember.AnalyzeNode(); return; } } } // call the base class to recurse base.AnalyzeNode(); // call this AFTER recursing to give the fields a chance to resolve, because we only // want to make this replacement if we are working on the global Date object. if (!m_inBrackets && !m_isConstructor && (m_args == null || m_args.Count == 0) && member != null && string.CompareOrdinal(member.Name, "getTime") == 0 && Parser.Settings.IsModificationAllowed(TreeModifications.DateGetTimeToUnaryPlus)) { // this is not a constructor and it's not a brackets call, and there are no arguments. // if the function is a member operation to "getTime" and the object of the member is a // constructor call to the global "Date" object (not a local), then we want to replace the call // with a unary plus on the Date constructor. Converting to numeric type is the same as // calling getTime, so it's the equivalent with much fewer bytes. CallNode dateConstructor = member.Root as CallNode; if (dateConstructor != null && dateConstructor.IsConstructor) { // lookup for the predifined (not local) "Date" field Lookup lookup = dateConstructor.Function as Lookup; if (lookup != null && string.CompareOrdinal(lookup.Name, "Date") == 0 && lookup.LocalField == null) { // this is in the pattern: (new Date()).getTime() // we want to replace it with +new Date // use the same date constructor node as the operand NumericUnary unary = new NumericUnary(Context, Parser, dateConstructor, JSToken.Plus); // replace us (the call to the getTime method) with this unary operator Parent.ReplaceChild(this, unary); // don't need to AnalyzeNode on the unary operator. The operand has already // been analyzed when we recursed, and the unary operator wouldn't do anything // special anyway (since the operand is not a numeric constant) } } } else if (Parser.Settings.EvalTreatment != EvalTreatment.Ignore) { // if this is a window.eval call, then we need to mark this scope as unknown just as // we would if this was a regular eval call. // (unless, of course, the parser settings say evals are safe) // call AFTER recursing so we know the left-hand side properties have had a chance to // lookup their fields to see if they are local or global if (member != null && string.CompareOrdinal(member.Name, "eval") == 0) { if (member.LeftHandSide.IsWindowLookup) { // this is a call to window.eval() // mark this scope as unknown so we don't crunch out locals // we might reference in the eval at runtime ScopeStack.Peek().IsKnownAtCompileTime = false; } } else { CallNode callNode = m_func as CallNode; if (callNode != null && callNode.InBrackets && callNode.LeftHandSide.IsWindowLookup && callNode.Arguments.IsSingleConstantArgument("eval")) { // this is a call to window["eval"] // mark this scope as unknown so we don't crunch out locals // we might reference in the eval at runtime ScopeStack.Peek().IsKnownAtCompileTime = false; } } } /* REVIEW: may be too late. lookups may alread have been analyzed and * found undefined * // check to see if this is an assignment to a window["prop"] structure * BinaryOperator binaryOp = Parent as BinaryOperator; * if (binaryOp != null && binaryOp.IsAssign * && m_inBrackets * && m_func.IsWindowLookup * && m_args != null) * { * // and IF the property is a non-empty constant that isn't currently * // a global field... * string propertyName = m_args.SingleConstantArgument; * if (!string.IsNullOrEmpty(propertyName) * && Parser.GlobalScope[propertyName] == null) * { * // we want to also add it to the global fields so it's not undefined * Parser.GlobalScope.DeclareField(propertyName, null, 0); * } * } */ }
private JSScanner GetJSScanner() { if (_scanner == null) { _scanner = new JSScanner(new CodeSettings() { AllowShebangLine = true }); } return _scanner; }
private LineTokenization TokenizeLine(JSScanner JSScanner, ITextSnapshot snapshot, object previousLineState, int lineNo) { ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo); SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak); var tcp = new SnapshotSpanSourceCodeReader(lineSpan); JSScanner.Initialize( lineSpan.GetText(), previousLineState, new SourceLocation(0, lineNo + 1, 1) ); try { var tokens = JSScanner.ReadTokens(lineSpan.Length).Select(ToTokenKind).ToArray(); return new LineTokenization(tokens, JSScanner.CurrentState); } finally { JSScanner.Uninitialize(); } }
/// <summary> /// Rescans the part of the buffer affected by a change. /// Scans a contiguous sub-<paramref name="span"/> of a larger code span which starts at <paramref name="codeStartLine"/>. /// </summary> private void ApplyChange(JSScanner JSScanner, ITextSnapshot snapshot, Span span) { int firstLine = snapshot.GetLineNumberFromPosition(span.Start); int lastLine = snapshot.GetLineNumberFromPosition(span.Length > 0 ? span.End - 1 : span.End); Contract.Assert(firstLine >= 0); // find the closest line preceding firstLine for which we know categorizer state, stop at the codeStartLine: LineTokenization lineTokenization; firstLine = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenization) + 1; object state = lineTokenization.State; int currentLine = firstLine; object previousState; while (currentLine < snapshot.LineCount) { previousState = _tokenCache.TryGetTokenization(currentLine, out lineTokenization) ? lineTokenization.State : null; _tokenCache[currentLine] = lineTokenization = TokenizeLine(JSScanner, snapshot, state, currentLine); state = lineTokenization.State; // stop if we visted all affected lines and the current line has no tokenization state or its previous state is the same as the new state: if (currentLine > lastLine && (previousState == null || previousState.Equals(state))) { break; } currentLine++; } // classification spans might have changed between the start of the first and end of the last visited line: int changeStart = snapshot.GetLineFromLineNumber(firstLine).Start; int changeEnd = (currentLine < snapshot.LineCount) ? snapshot.GetLineFromLineNumber(currentLine).End : snapshot.Length; if (changeStart < changeEnd) { var classificationChanged = ClassificationChanged; if (classificationChanged != null) { var args = new ClassificationChangedEventArgs(new SnapshotSpan(snapshot, new Span(changeStart, changeEnd - changeStart))); classificationChanged(this, args); } } }