public bool AddDebugLookup(string debugNamespace)
{
// a blank identifier is okay -- we just ignore it
if (!string.IsNullOrEmpty(debugNamespace))
{
// but if it's not blank, it better be a valid JavaScript identifier or member chain
if (debugNamespace.IndexOf('.') > 0)
{
// it's a member chain -- check that each part is a valid JS identifier
var names = debugNamespace.Split('.');
foreach (var name in names)
{
if (!JSScanner.IsValidIdentifier(name))
{
return(false);
}
}
}
else
{
// no dot -- just an identifier
if (!JSScanner.IsValidIdentifier(debugNamespace))
{
return(false);
}
}
m_debugLookups.Add(debugNamespace);
return(true);
}
return(false);
}
public override string ToCode(ToCodeFormat format)
{
StringBuilder sb = new StringBuilder();
sb.Append("do");
ToCodeFormat bodyFormat = ((Body != null &&
Body.Count == 1 &&
Body[0].GetType() == typeof(DoWhile))
? ToCodeFormat.AlwaysBraces
: ToCodeFormat.Normal
);
// if the body is a single statement that ends in a do-while, then we
// will need to wrap the body in curly-braces to get around an IE bug
if (Body != null && Body.EncloseBlock(EncloseBlockType.SingleDoWhile))
{
bodyFormat = ToCodeFormat.AlwaysBraces;
}
string bodyString = (
Body == null
? string.Empty
: Body.ToCode(bodyFormat)
);
if (bodyString.Length == 0)
{
sb.Append(';');
}
else
{
// if the first character could be interpreted as a continuation
// of the "do" keyword, then we need to add a space
if (JSScanner.StartsWithIdentifierPart(bodyString))
{
sb.Append(' ');
}
sb.Append(bodyString);
// if there is no body, we need a semi-colon
// OR if we didn't always wrap in braces AND we require a separator, we need a semi-colon.
// and make sure it doesn't already end in a semicolon -- we don't want two in a row.
if (Body == null ||
(bodyFormat != ToCodeFormat.AlwaysBraces && Body.RequiresSeparator && !bodyString.EndsWith(";", StringComparison.Ordinal)))
{
sb.Append(';');
}
}
// add a space for readability for pretty-print mode
if (Parser.Settings.OutputMode == OutputMode.MultipleLines && Parser.Settings.IndentSize > 0)
{
sb.Append(' ');
}
sb.Append("while(");
sb.Append(Condition.ToCode());
sb.Append(")");
return(sb.ToString());
}
/// <summary>
/// Add a known global identifier to the list
/// </summary>
/// <param name="identifier">global identifier</param>
/// <returns>true if valid identifier; false if invalid identifier</returns>
public bool AddKnownGlobal(string identifier)
{
if (JSScanner.IsValidIdentifier(identifier))
{
m_knownGlobals.Add(identifier);
return(true);
}
return(false);
}
public bool AddNoAutoRename(string noRename)
{
if (!JSScanner.IsValidIdentifier(noRename))
{
return(false);
}
m_noRenameSet.Add(noRename);
return(true);
}
private JSScanner GetJSScanner()
{
if (_scanner == null)
{
_scanner = new JSScanner(new CodeSettings()
{
AllowShebangLine = true
});
}
return(_scanner);
}
private static List <TokenWithSpan> ReadTokens(string code, bool collectWarnings, params ErrorInfo[] errors)
{
CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings);
var scanner = new JSScanner(code, errorSink, new CodeSettings()
{
AllowShebangLine = true
});
var tokens = scanner.ReadTokens(Int32.MaxValue);
errorSink.CheckErrors(errors);
return(tokens);
}
internal string NextName()
{
string name;
do
{
// advance to the next name
++m_currentName;
name = CurrentName;
// keep advancing until we find one that isn't in the skip list or a keyword
}while (m_skipNames.ContainsKey(name) || JSScanner.IsKeyword(name));
return(name);
}
// Determine if an identifier is valid for this engine.
public override bool IsValidIdentifier(String identifier)
{
if (identifier == null)
{
return(false);
}
JSScanner scanner = new JSScanner(identifier);
if (scanner.FetchIdentifier() == null)
{
return(false);
}
return(scanner.Fetch() == -1);
}
public override string ToCode(ToCodeFormat format)
{
StringBuilder sb = new StringBuilder();
sb.Append("return");
if (m_operand != null)
{
string operandText = m_operand.ToCode();
if (operandText.Length > 0 && JSScanner.StartsWithIdentifierPart(operandText))
{
sb.Append(' ');
}
sb.Append(operandText);
}
return(sb.ToString());
}
public override string ToCode(ToCodeFormat format)
{
if (PrimitiveType == PrimitiveType.String)
{
string rawValue = Value.ToString();
// if the raw value is safe to be an identifier, then go ahead and ditch the quotes and just output
// the raw value. Otherwise call ToCode to wrap the string in quotes.
return(JSScanner.IsSafeIdentifier(rawValue) && !JSScanner.IsKeyword(rawValue) ? rawValue : base.ToCode(format));
}
else
{
// call the base to format the value
return(base.ToCode(format));
}
}
/// <summary>
/// Set the dictionary of preprocessor defines and values
/// </summary>
/// <param name="defines">dictionary to set</param>
public int SetPreprocessorValues(IDictionary <string, string> defines)
{
PreprocessorValues.Clear();
if (defines != null && defines.Count > 0)
{
foreach (var define in defines)
{
if (JSScanner.IsValidIdentifier(define.Key))
{
PreprocessorValues.Add(define.Key, define.Value);
}
}
}
return(PreprocessorValues.Count);
}
private static bool NeedsParens(AstNode node, JSToken refToken)
{
bool needsParens = false;
// assignments and commas are the only operators that need parens
// around them. Conditional is pretty low down the list
BinaryOperator binaryOp = node as BinaryOperator;
if (binaryOp != null)
{
OpPrec thisPrecedence = JSScanner.GetOperatorPrecedence(refToken);
OpPrec nodePrecedence = JSScanner.GetOperatorPrecedence(binaryOp.OperatorToken);
needsParens = (nodePrecedence < thisPrecedence);
}
return(needsParens);
}
/// <summary>
/// Rescans the part of the buffer affected by a change.
/// Scans a contiguous sub-<paramref name="span"/> of a larger code span which starts at <paramref name="codeStartLine"/>.
/// </summary>
private void ApplyChange(JSScanner JSScanner, ITextSnapshot snapshot, Span span)
{
int firstLine = snapshot.GetLineNumberFromPosition(span.Start);
int lastLine = snapshot.GetLineNumberFromPosition(span.Length > 0 ? span.End - 1 : span.End);
Contract.Assert(firstLine >= 0);
// find the closest line preceding firstLine for which we know categorizer state, stop at the codeStartLine:
LineTokenization lineTokenization;
firstLine = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenization) + 1;
object state = lineTokenization.State;
int currentLine = firstLine;
object previousState;
while (currentLine < snapshot.LineCount)
{
previousState = _tokenCache.TryGetTokenization(currentLine, out lineTokenization) ? lineTokenization.State : null;
_tokenCache[currentLine] = lineTokenization = TokenizeLine(JSScanner, snapshot, state, currentLine);
state = lineTokenization.State;
// stop if we visted all affected lines and the current line has no tokenization state or its previous state is the same as the new state:
if (currentLine > lastLine && (previousState == null || previousState.Equals(state)))
{
break;
}
currentLine++;
}
// classification spans might have changed between the start of the first and end of the last visited line:
int changeStart = snapshot.GetLineFromLineNumber(firstLine).Start;
int changeEnd = (currentLine < snapshot.LineCount) ? snapshot.GetLineFromLineNumber(currentLine).End : snapshot.Length;
if (changeStart < changeEnd)
{
var classificationChanged = ClassificationChanged;
if (classificationChanged != null)
{
var args = new ClassificationChangedEventArgs(new SnapshotSpan(snapshot, new Span(changeStart, changeEnd - changeStart)));
classificationChanged(this, args);
}
}
}
private static List <TokenWithSpan> ScanTokens(string code, bool collectWarnings, params ErrorInfo[] errors)
{
CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings);
var scanner = new JSScanner(code, errorSink, new CodeSettings()
{
AllowShebangLine = true
});
List <TokenWithSpan> tokens = new List <TokenWithSpan>();
for (TokenWithSpan curToken = scanner.ScanNextTokenWithSpan(true);
curToken.Token != JSToken.EndOfFile;
curToken = scanner.ScanNextTokenWithSpan(true))
{
tokens.Add(curToken);
}
errorSink.CheckErrors(errors);
return(tokens);
}
// Test a single operator.
private void TestOp(JSScanner scanner, String ops,
ref int posn, JSToken expected)
{
Context token = JSScannerTest.TestGetTokenContext(scanner);
scanner.GetNextToken();
int next = ops.IndexOf(' ', posn);
String thisop = ops.Substring(posn, next - posn);
AssertEquals("TestOp[" + thisop + "] (1)",
expected, token.GetToken());
AssertEquals("TestOp[" + thisop + "] (2)",
thisop, token.GetCode());
AssertEquals("TestOp[" + thisop + "] (3)",
posn, token.StartPosition);
AssertEquals("TestOp[" + thisop + "] (4)",
next, token.EndPosition);
posn = next + 1;
}
private LineTokenization TokenizeLine(JSScanner JSScanner, ITextSnapshot snapshot, object previousLineState, int lineNo)
{
ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo);
SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak);
var tcp = new SnapshotSpanSourceCodeReader(lineSpan);
JSScanner.Initialize(
lineSpan.GetText(),
previousLineState,
new SourceLocation(0, lineNo + 1, 1)
);
try {
var tokens = JSScanner.ReadTokens(lineSpan.Length).Select(ToTokenKind).ToArray();
return(new LineTokenization(tokens, JSScanner.CurrentState));
} finally {
JSScanner.Uninitialize();
}
}
public static string CrunchedLabel(int nestLevel)
{
// nestCount is 0-based.
// return null if the nestLevel is invalid (< 0).
string minLabel = null;
if (nestLevel >= 0)
{
minLabel = GenerateNameFromNumber(nestLevel);
if (JSScanner.IsKeyword(minLabel, true))
{
// prepend something to make it NOT a keyword.
// no keywords start with an underscore, so...
minLabel = '_' + minLabel;
}
}
return(minLabel);
}
private static bool IsOnlyDecimalDigits(string text)
{
// if text is null, return false.
// Otherwise return true if ALL the characters are decimal digits,
// or false is ANY ONE character isn't.
//return text.IfNotNull(s => !s.Any(c => !JSScanner.IsDigit(c)));
return(text.IfNotNull(s =>
{
foreach (char c in s)
{
if (!JSScanner.IsDigit(c))
{
return false;
}
}
return true;
}));
}
public override string ToCode(ToCodeFormat format)
{
string operandString = Operand.ToCode(format);
if (NeedsParentheses)
{
// needs parens
return("delete(" + operandString + ')');
}
else if (JSScanner.StartsWithIdentifierPart(operandString))
{
// needs a space
return("delete " + operandString);
}
else
{
// needs no separator
return("delete" + operandString);
}
}
public override string ToCode(ToCodeFormat format)
{
string operandString = Operand.ToCode(format);
if (NeedsParentheses)
{
// need parentheses
return("typeof(" + operandString + ')');
}
else if (JSScanner.StartsWithIdentifierPart(operandString))
{
// need a space separating them
return("typeof " + operandString);
}
else
{
// don't need the space
return("typeof" + operandString);
}
}
public override string ToCode(ToCodeFormat format)
{
StringBuilder sb = new StringBuilder();
// the label should be indented
Parser.Settings.Indent();
// start a new line
Parser.Settings.NewLine(sb);
if (m_caseValue != null)
{
sb.Append("case");
string caseValue = m_caseValue.ToCode();
if (JSScanner.StartsWithIdentifierPart(caseValue))
{
sb.Append(' ');
}
sb.Append(caseValue);
}
else
{
sb.Append("default");
}
sb.Append(':');
// in pretty-print mode, we indent the statements under the label, too
Parser.Settings.Indent();
// output the statements
if (m_statements != null && m_statements.Count > 0)
{
sb.Append(m_statements.ToCode(ToCodeFormat.NoBraces));
}
// if we are pretty-printing, we need to unindent twice:
// once for the label, and again for the statements
Parser.Settings.Unindent();
Parser.Settings.Unindent();
return(sb.ToString());
}
private void UnreferencedFunction(JSVariableField variableField, FunctionObject functionObject)
{
// if there is no name, then ignore this declaration because it's malformed.
// (won't be a function expression because those are automatically referenced).
// also ignore ghosted function fields.
if (functionObject.Name != null && variableField.FieldType != FieldType.GhostFunction)
{
// if the function name isn't a simple identifier, then leave it there and mark it as
// not renamable because it's probably one of those darn IE-extension event handlers or something.
if (JSScanner.IsValidIdentifier(functionObject.Name))
{
// unreferenced function declaration. fire a warning.
var ctx = functionObject.IdContext ?? variableField.OriginalContext;
ctx.HandleError(JSError.FunctionNotReferenced, false);
// hide it from the output if our settings say we can.
// we don't want to delete it, per se, because we still want it to
// show up in the scope report so the user can see that it was unreachable
// in case they are wondering where it went.
// ES6 has the notion of block-scoped function declarations. ES5 says functions can't
// be defined inside blocks -- only at the root level of the global scope or function scopes.
// so if this is a block scope, don't hide the function, even if it is unreferenced because
// of the cross-browser difference.
if (this.IsKnownAtCompileTime &&
m_settings.MinifyCode &&
m_settings.RemoveUnneededCode &&
!(this is BlockScope))
{
functionObject.HideFromOutput = true;
}
}
else
{
// not a valid identifier name for this function. Don't rename it because it's
// malformed and we don't want to mess up the developer's intent.
variableField.CanCrunch = false;
}
}
}
// Determine if a namespace name is valid.
protected override bool IsValidNamespaceName(String name)
{
if (name == null)
{
return(false);
}
JSScanner scanner = new JSScanner(name);
if (scanner.FetchIdentifier() == null)
{
return(false);
}
while (scanner.Peek() == '.')
{
scanner.Fetch();
if (scanner.FetchIdentifier() == null)
{
return(false);
}
}
return(scanner.Fetch() == -1);
}
public void PartialScanning() {
var code1 = "/* hello world ";
var code2 = " goodbye */";
CollectingErrorSink errorSink = new CollectingErrorSink(true);
var scanner = new JSScanner(code1, errorSink, new CodeSettings() { AllowShebangLine = true });
var tokens = scanner.ReadTokens(Int32.MaxValue);
VerifyTokens(
tokens,
new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(0, 1, 1), new SourceLocation(15, 1, 16))
);
scanner.Initialize(code2, scanner.CurrentState, new SourceLocation(code1.Length, 2, 1));
tokens = scanner.ReadTokens(Int32.MaxValue);
VerifyTokens(
tokens,
new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(15, 2, 16), new SourceLocation(28, 2, 29))
);
Assert.IsTrue(scanner.CurrentState.Equals(scanner.CurrentState));
Assert.IsFalse(scanner.CurrentState.Equals(new object()));
Assert.AreEqual(scanner.CurrentState.GetHashCode(), 2);
}
/// <summary>
/// We really only care about caching strings that are likely
/// to be identifiers. Everything else we'll coalesce into the
/// empty string.
/// </summary>
private static bool DontCacheString(string strValue)
{
if (strValue.Length > 100)
{
return(true);
}
for (int i = 0; i < strValue.Length; i++)
{
char ch = strValue[i];
if (ch == '-' || ch == '.' || ch == '/' || ch == '\\')
{
// not valid identifiers, but likely to show up in
// require string literals, so we still care about preserving
// these strings.
continue;
}
if (!JSScanner.IsValidIdentifierPart(strValue[i]))
{
return(true);
}
}
return(false);
}
public bool Match(AstNode node, string identifiers)
{
// set the match to false
m_isMatch = false;
// identifiers cannot be null or blank and must match: IDENT(.IDENT)*
// since for JS there has to be at least a global object, the dot must be AFTER the first character.
if (node != null && !string.IsNullOrEmpty(identifiers))
{
// get all the parts
var parts = identifiers.Split('.');
// each part must be a valid JavaScript identifier. Assume everything is valid
// unless at least one is invalid -- then forget it
var isValid = true;
foreach (var part in parts)
{
if (!JSScanner.IsValidIdentifier(part))
{
isValid = false;
break;
}
}
// must be valid to continue
if (isValid)
{
// save the parts and start the index on the last one, since we'll be walking backwards
m_parts = parts;
m_index = parts.Length - 1;
node.Accept(this);
}
}
return(m_isMatch);
}
/// <summary>
/// Add a rename pair to the identifier rename map
/// </summary>
/// <param name="sourceName">name of the identifier in the source code</param>
/// <param name="newName">new name with which to replace the source name</param>
/// <returns>true if added; false if either name is not a valid JavaScript identifier</returns>
public bool AddRenamePair(string sourceName, string newName)
{
bool successfullyAdded = false;
// both names MUST be valid JavaScript identifiers
if (JSScanner.IsValidIdentifier(sourceName) && JSScanner.IsValidIdentifier(newName))
{
if (m_identifierReplacementMap.ContainsKey(sourceName))
{
// just replace the value
m_identifierReplacementMap[sourceName] = newName;
}
else
{
// add the new pair
m_identifierReplacementMap.Add(sourceName, newName);
}
// if we get here, we added it (or updated it if it's a dupe)
successfullyAdded = true;
}
return(successfullyAdded);
}
private LineTokenization GetPreviousTokenization(JSScanner JSScanner, ITextSnapshot snapshot, int firstLine, int prevLine) {
LineTokenization prevLineTokenization;
if (!_tokenCache.TryGetTokenization(prevLine, out prevLineTokenization)) {
LineTokenization lineTokenizationTemp;
int currentLineTemp = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenizationTemp) + 1;
object stateTemp = lineTokenizationTemp.State;
while (currentLineTemp < snapshot.LineCount) {
if (!_tokenCache.TryGetTokenization(currentLineTemp, out lineTokenizationTemp)) {
lineTokenizationTemp = TokenizeLine(JSScanner, snapshot, stateTemp, currentLineTemp);
_tokenCache[currentLineTemp] = lineTokenizationTemp;
}
currentLineTemp++;
stateTemp = lineTokenizationTemp.State;
}
prevLineTokenization = TokenizeLine(JSScanner, snapshot, stateTemp, prevLine);
_tokenCache[prevLine] = prevLineTokenization;
}
return prevLineTokenization;
}
static string CreateJSFromResourceStrings(ResourceStrings resourceStrings)
{
var sb = StringBuilderPool.Acquire();
try
{
// start the var statement using the requested name and open the initializer object literal
sb.Append("var ");
sb.Append(resourceStrings.Name);
sb.Append("={");
// we're going to need to insert commas between each pair, so we'll use a boolean
// flag to indicate that we're on the first pair. When we output the first pair, we'll
// set the flag to false. When the flag is false, we're about to insert another pair, so
// we'll add the comma just before.
bool firstItem = true;
// loop through all items in the collection
foreach (var keyPair in resourceStrings.NameValuePairs)
{
// if this isn't the first item, we need to add a comma separator
if (!firstItem)
{
sb.Append(',');
}
else
{
// next loop is no longer the first item
firstItem = false;
}
// append the key as the name, a colon to separate the name and value,
// and then the value
// must quote if not valid JS identifier format, or if it is, but it's a keyword
// (use strict mode just to be safe)
string propertyName = keyPair.Key;
if (!JSScanner.IsValidIdentifier(propertyName) || JSScanner.IsKeyword(propertyName, true))
{
sb.Append("\"");
// because we are using quotes for the delimiters, replace any instances
// of a quote character (") with an escaped quote character (\")
sb.Append(propertyName.Replace("\"", "\\\""));
sb.Append("\"");
}
else
{
sb.Append(propertyName);
}
sb.Append(':');
// make sure the Value is properly escaped, quoted, and whatever we
// need to do to make sure it's a proper JS string.
// pass false for whether this string is an argument to a RegExp constructor.
// pass false for whether to use W3Strict formatting for character escapes (use maximum browser compatibility)
// pass true for ecma strict mode
string stringValue = ConstantWrapper.EscapeString(
keyPair.Value,
false,
false,
true
);
sb.Append(stringValue);
}
// close the object literal and return the string
sb.AppendLine("};");
return(sb.ToString());
}
finally
{
sb.Release();
}
}
/// <summary>
/// Creates an instance of the JSParser class that can be used to parse the given source code.
/// </summary>
/// <param name="source">Source code to parse.</param>
public JSParser(string source)
{
m_severity = 5;
m_blockType = new List<BlockType>(16);
m_labelTable = new Dictionary<string, LabelInfo>();
m_noSkipTokenSet = new NoSkipTokenSet();
m_importantComments = new List<Context>();
m_document = new DocumentContext(this, source);
m_scanner = new JSScanner(new Context(m_document));
m_currentToken = new Context(m_document);
// if the scanner encounters a special "globals" comment, it'll fire this event
// at which point we will define a field with that name in the global scope.
m_scanner.GlobalDefine += (sender, ea) =>
{
var globalScope = GlobalScope;
if (globalScope[ea.Name] == null)
{
var field = globalScope.CreateField(ea.Name, null, FieldAttributes.SpecialName);
globalScope.AddField(field);
}
};
// this event is fired whenever a ///#SOURCE comment is encountered
m_scanner.NewModule += (sender, ea) =>
{
m_newModule = true;
};
}
// Test a single operator.
private void TestOp(JSScanner scanner, String ops,
ref int posn, JSToken expected)
{
Context token = JSScannerTest.TestGetTokenContext(scanner);
scanner.GetNextToken();
int next = ops.IndexOf(' ', posn);
String thisop = ops.Substring(posn, next - posn);
AssertEquals("TestOp[" + thisop + "] (1)",
expected, token.GetToken());
AssertEquals("TestOp[" + thisop + "] (2)",
thisop, token.GetCode());
AssertEquals("TestOp[" + thisop + "] (3)",
posn, token.StartPosition);
AssertEquals("TestOp[" + thisop + "] (4)",
next, token.EndPosition);
posn = next + 1;
}
private static List<TokenWithSpan> ScanTokens(string code, bool collectWarnings, params ErrorInfo[] errors) {
CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings);
var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true });
List<TokenWithSpan> tokens = new List<TokenWithSpan>();
for (TokenWithSpan curToken = scanner.ScanNextTokenWithSpan(true);
curToken.Token != JSToken.EndOfFile;
curToken = scanner.ScanNextTokenWithSpan(true)) {
tokens.Add(curToken);
}
errorSink.CheckErrors(errors);
return tokens;
}
private static List<TokenWithSpan> ReadTokens(string code, bool collectWarnings, params ErrorInfo[] errors) {
CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings);
var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true });
var tokens = scanner.ReadTokens(Int32.MaxValue);
errorSink.CheckErrors(errors);
return tokens;
}
internal override Dictionary <string, IAnalysisSet> GetOwnProperties(ProjectEntry accessor)
{
if (_descriptors == null || _descriptors.Count == 0)
{
return(new Dictionary <string, IAnalysisSet>());
}
var res = new Dictionary <string, IAnalysisSet>();
foreach (var kvp in _descriptors)
{
var key = kvp.Key;
if (!JSScanner.IsValidIdentifier(key))
{
// https://nodejstools.codeplex.com/workitem/987
// for intellisense purposes we don't report invalid identifiers, but we can
// end up with these from indexing with constant strings.
continue;
}
if (kvp.Value.Values != null)
{
//kvp.Value.Values.ClearOldValues();
if (kvp.Value.Values.VariableStillExists)
{
var types = kvp.Value.Values.GetTypesNoCopy(accessor, ProjectEntry);
if (types.Count != 0 ||
kvp.Value.Values.TypesNoCopy.Count == 0)
{
MergeTypes(res, key, types);
}
}
}
if (kvp.Value.Getter != null)
{
foreach (var value in kvp.Value.Getter.GetTypesNoCopy(accessor, ProjectEntry))
{
FunctionValue userFunc = value.Value as FunctionValue;
if (userFunc != null)
{
MergeTypes(res, key, userFunc.ReturnTypes);
}
}
}
if (kvp.Value.Setter != null)
{
MergeTypes(res, key, AnalysisSet.Empty);
}
}
if (_linkedValues != null)
{
foreach (var linkedValue in _linkedValues.GetTypesNoCopy(accessor, ProjectEntry))
{
if (linkedValue.Value.Push())
{
try {
MergeDictionaries(res, linkedValue.Value.GetAllMembers(accessor));
} finally {
linkedValue.Value.Pop();
}
}
}
}
return(res);
}
/// <summary>
/// Adds classification spans to the given collection.
/// Scans a contiguous sub-<paramref name="span"/> of a larger code span which starts at <paramref name="codeStartLine"/>.
/// </summary>
private void AddClassifications(JSScanner JSScanner, List<ClassificationSpan> classifications, SnapshotSpan span) {
Debug.Assert(span.Length > 0);
var snapshot = span.Snapshot;
int firstLine = snapshot.GetLineNumberFromPosition(span.Start);
int lastLine = snapshot.GetLineNumberFromPosition(span.End - 1);
Contract.Assert(firstLine >= 0);
_tokenCache.EnsureCapacity(snapshot.LineCount);
// find the closest line preceding firstLine for which we know categorizer state, stop at the codeStartLine:
LineTokenization lineTokenization;
int currentLine = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenization) + 1;
object state = lineTokenization.State;
// track the previous 2 tokens to adjust our classifications of keywords
// when they shouldn't be displayed as keywords...
TokenInfoWithLine? prevToken = null, prevPrevToken = null;
// initialize the previous tokens so we can handle things like:
// foo.
// get()
// even if we're called on the line for get()
int prevLine = currentLine - 1;
while (prevLine >= 0 && prevToken == null) {
LineTokenization prevLineTokenization = GetPreviousTokenization(JSScanner, snapshot, firstLine, prevLine);
for (int i = prevLineTokenization.Tokens.Length - 1; i >= 0 && prevToken == null; i--) {
var tempToken = prevLineTokenization.Tokens[i];
if (IsValidPreviousToken(ref tempToken)) {
prevToken = prevPrevToken;
prevPrevToken = new TokenInfoWithLine() { TokenInfo = tempToken, Line = prevLine };
}
}
prevLine--;
}
while (currentLine <= lastLine) {
if (!_tokenCache.TryGetTokenization(currentLine, out lineTokenization)) {
lineTokenization = TokenizeLine(JSScanner, snapshot, state, currentLine);
_tokenCache[currentLine] = lineTokenization;
}
state = lineTokenization.State;
for (int i = 0; i < lineTokenization.Tokens.Length; i++) {
var token = lineTokenization.Tokens[i];
if (token.Category == TokenCategory.IncompleteMultiLineStringLiteral || token.Category == TokenCategory.Comment) {
IClassificationType type;
switch (token.Category) {
case TokenCategory.IncompleteMultiLineStringLiteral:
type = _provider.StringLiteral;
break;
case TokenCategory.Comment:
type = _provider.Comment;
break;
default:
type = null;
break;
}
Debug.Assert(type != null, "We should have a defined ClassificationType for every token.");
// we need to walk backwards to find the start of this multi-line string...
TokenInfo startToken = token;
int validPrevLine;
int length = startToken.SourceSpan.Length;
if (i == 0) {
length += GetLeadingMultiLineTokens(JSScanner, snapshot, token.Category, firstLine, currentLine, out validPrevLine, ref startToken);
} else {
validPrevLine = currentLine;
}
if (i == lineTokenization.Tokens.Length - 1) {
length += GetTrailingMultiLineTokens(JSScanner, snapshot, token.Category, currentLine, state);
}
var tokenSpan = new Span(SnapshotSpanToSpan(snapshot, startToken, validPrevLine).Start, length);
var intersection = span.Intersection(tokenSpan);
if ((intersection != null && intersection.Value.Length > 0) ||
(span.Length == 0 && tokenSpan.Contains(span.Start)) // handle zero-length spans
) {
classifications.Add(new ClassificationSpan(new SnapshotSpan(snapshot, tokenSpan), type));
}
} else {
ClassificationSpan classification = null;
if (token.Category == TokenCategory.Keyword) {
// check and see if we're not really a keyword...
if (IsKeywordInIdentifierContext(snapshot, prevToken, prevPrevToken, new TokenInfoWithLine() { TokenInfo = token, Line = currentLine })) {
classification = GetClassificationSpan(
span,
token,
currentLine,
CategoryMap[TokenCategory.Identifier]
);
}
}
if (classification == null) {
classification = ClassifyToken(span, token, currentLine);
}
if (classification != null) {
classifications.Add(classification);
}
}
if (IsValidPreviousToken(ref token)) {
prevPrevToken = prevToken;
prevToken = new TokenInfoWithLine() { TokenInfo = token, Line = currentLine };
}
}
currentLine++;
}
}
private int GetLeadingMultiLineTokens(JSScanner JSScanner, ITextSnapshot snapshot, TokenCategory tokenCategory, int firstLine, int currentLine, out int validPrevLine, ref TokenInfo startToken) {
validPrevLine = currentLine;
int prevLine = currentLine - 1;
int length = 0;
while (prevLine >= 0) {
LineTokenization prevLineTokenization = GetPreviousTokenization(JSScanner, snapshot, firstLine, prevLine);
if (prevLineTokenization.Tokens.Length != 0) {
if (prevLineTokenization.Tokens[prevLineTokenization.Tokens.Length - 1].Category != tokenCategory) {
break;
}
startToken = prevLineTokenization.Tokens[prevLineTokenization.Tokens.Length - 1];
length += startToken.SourceSpan.Length;
}
validPrevLine = prevLine;
prevLine--;
if (prevLineTokenization.Tokens.Length > 1) {
// http://pytools.codeplex.com/workitem/749
// if there are multiple tokens on this line then our multi-line string
// is terminated.
break;
}
}
return length;
}
public override string ToCode(ToCodeFormat format)
{
StringBuilder sb = new StringBuilder();
sb.Append("if(");
sb.Append(Condition.ToCode());
sb.Append(')');
// if we're in Safari-quirks mode, we will need to wrap the if block
// in curly braces if it only includes a function declaration. Safari
// throws parsing errors in those situations
ToCodeFormat elseFormat = ToCodeFormat.Normal;
if (FalseBlock != null && FalseBlock.Count == 1)
{
if (Parser.Settings.MacSafariQuirks &&
FalseBlock[0] is FunctionObject)
{
elseFormat = ToCodeFormat.AlwaysBraces;
}
else if (FalseBlock[0] is IfNode)
{
elseFormat = ToCodeFormat.ElseIf;
}
}
// get the else block -- we need to know if there is anything in order
// to fully determine if the true-branch needs curly-braces
string elseBlock = (
FalseBlock == null
? string.Empty
: FalseBlock.ToCode(elseFormat));
// we'll need to force the true block to be enclosed in curly braces if
// there is an else block and the true block contains a single statement
// that ends in an if that doesn't have an else block
ToCodeFormat trueFormat = (FalseBlock != null &&
TrueBlock != null &&
TrueBlock.EncloseBlock(EncloseBlockType.IfWithoutElse)
? ToCodeFormat.AlwaysBraces
: ToCodeFormat.Normal);
if (elseBlock.Length > 0 &&
TrueBlock != null &&
TrueBlock.EncloseBlock(EncloseBlockType.SingleDoWhile))
{
trueFormat = ToCodeFormat.AlwaysBraces;
}
// if we're in Safari-quirks mode, we will need to wrap the if block
// in curly braces if it only includes a function declaration. Safari
// throws parsing errors in those situations
if (Parser.Settings.MacSafariQuirks &&
TrueBlock != null &&
TrueBlock.Count == 1 &&
TrueBlock[0] is FunctionObject)
{
trueFormat = ToCodeFormat.AlwaysBraces;
}
// add the true block
string trueBlock = (
TrueBlock == null
? string.Empty
: TrueBlock.ToCode(trueFormat));
sb.Append(trueBlock);
if (elseBlock.Length > 0)
{
if (trueFormat != ToCodeFormat.AlwaysBraces &&
!trueBlock.EndsWith(";", StringComparison.Ordinal) &&
(TrueBlock == null || TrueBlock.RequiresSeparator))
{
sb.Append(';');
}
// if we are in pretty-print mode, drop the else onto a new line
Parser.Settings.NewLine(sb);
sb.Append("else");
// if the first character could be interpreted as a continuation
// of the "else" statement, then we need to add a space
if (JSScanner.StartsWithIdentifierPart(elseBlock))
{
sb.Append(' ');
}
sb.Append(elseBlock);
}
return(sb.ToString());
}
private int GetTrailingMultiLineTokens(JSScanner JSScanner, ITextSnapshot snapshot, TokenCategory tokenCategory, int currentLine, object state) {
int nextLine = currentLine + 1;
var prevState = state;
int length = 0;
while (nextLine < snapshot.LineCount) {
LineTokenization nextLineTokenization;
if (!_tokenCache.TryGetTokenization(nextLine, out nextLineTokenization)) {
nextLineTokenization = TokenizeLine(JSScanner, snapshot, prevState, nextLine);
prevState = nextLineTokenization.State;
_tokenCache[nextLine] = nextLineTokenization;
}
if (nextLineTokenization.Tokens.Length != 0) {
if (nextLineTokenization.Tokens[0].Category != tokenCategory) {
break;
}
length += nextLineTokenization.Tokens[0].SourceSpan.Length;
}
nextLine++;
}
return length;
}
internal override void AnalyzeNode()
{
// see if this is a member (we'll need it for a couple checks)
Member member = m_func as Member;
if (Parser.Settings.StripDebugStatements && Parser.Settings.IsModificationAllowed(TreeModifications.StripDebugStatements))
{
// if this is a member, and it's a debugger object, and it's a constructor....
if (member != null && member.IsDebuggerStatement && m_isConstructor)
{
// we need to replace our debugger object with a generic Object
m_func = new Lookup("Object", m_func.Context, Parser);
// and make sure the node list is empty
if (m_args != null && m_args.Count > 0)
{
m_args = new AstNodeList(m_args.Context, Parser);
}
}
}
// if this is a constructor and we want to collapse
// some of them to literals...
if (m_isConstructor && Parser.Settings.CollapseToLiteral)
{
// see if this is a lookup, and if so, if it's pointing to one
// of the two constructors we want to collapse
Lookup lookup = m_func as Lookup;
if (lookup != null)
{
if (lookup.Name == "Object" &&
Parser.Settings.IsModificationAllowed(TreeModifications.NewObjectToObjectLiteral))
{
// no arguments -- the Object constructor with no arguments is the exact same as an empty
// object literal
if (m_args == null || m_args.Count == 0)
{
// replace our node with an object literal
ObjectLiteral objLiteral = new ObjectLiteral(Context, Parser, null, null);
if (Parent.ReplaceChild(this, objLiteral))
{
// and bail now. No need to recurse -- it's an empty literal
return;
}
}
else if (m_args.Count == 1)
{
// one argument
// check to see if it's an object literal.
ObjectLiteral objectLiteral = m_args[0] as ObjectLiteral;
if (objectLiteral != null)
{
// the Object constructor with an argument that is a JavaScript object merely returns the
// argument. Since the argument is an object literal, it is by definition a JavaScript object
// and therefore we can replace the constructor call with the object literal
Parent.ReplaceChild(this, objectLiteral);
// don't forget to recurse the object now
objectLiteral.AnalyzeNode();
// and then bail -- we don't want to process this call
// operation any more; we've gotten rid of it
return;
}
}
}
else if (lookup.Name == "Array" &&
Parser.Settings.IsModificationAllowed(TreeModifications.NewArrayToArrayLiteral))
{
// Array is trickier.
// If there are no arguments, then just use [].
// if there are multiple arguments, then use [arg0,arg1...argN].
// but if there is one argument and it's numeric, we can't crunch it.
// also can't crunch if it's a function call or a member or something, since we won't
// KNOW whether or not it's numeric.
//
// so first see if it even is a single-argument constant wrapper.
ConstantWrapper constWrapper = (m_args != null && m_args.Count == 1 ? m_args[0] as ConstantWrapper : null);
// if the argument count is not one, then we crunch.
// if the argument count IS one, we only crunch if we have a constant wrapper,
// AND it's not numeric.
if (m_args == null ||
m_args.Count != 1 ||
(constWrapper != null && !constWrapper.IsNumericLiteral))
{
// create the new array literal object
ArrayLiteral arrayLiteral = new ArrayLiteral(Context, Parser, m_args);
// replace ourself within our parent
if (Parent.ReplaceChild(this, arrayLiteral))
{
// recurse
arrayLiteral.AnalyzeNode();
// and bail -- we don't want to recurse this node any more
return;
}
}
}
}
}
// if we are replacing resource references with strings generated from resource files
// and this is a brackets call: lookup[args]
ResourceStrings resourceStrings = Parser.ResourceStrings;
if (m_inBrackets && resourceStrings != null && resourceStrings.Count > 0)
{
// see if the root object is a lookup that corresponds to the
// global value (not a local field) for our resource object
// (same name)
Lookup rootLookup = m_func as Lookup;
if (rootLookup != null &&
rootLookup.LocalField == null &&
string.CompareOrdinal(rootLookup.Name, resourceStrings.Name) == 0)
{
// we're going to replace this node with a string constant wrapper
// but first we need to make sure that this is a valid lookup.
// if the parameter contains anything that would vary at run-time,
// then we need to throw an error.
// the parser will always have either one or zero nodes in the arguments
// arg list. We're not interested in zero args, so just make sure there is one
if (m_args.Count == 1)
{
// must be a constant wrapper
ConstantWrapper argConstant = m_args[0] as ConstantWrapper;
if (argConstant != null)
{
string resourceName = argConstant.Value.ToString();
// get the localized string from the resources object
ConstantWrapper resourceLiteral = new ConstantWrapper(
resourceStrings[resourceName],
PrimitiveType.String,
Context,
Parser);
// replace this node with localized string, analyze it, and bail
// so we don't anaylze the tree we just replaced
Parent.ReplaceChild(this, resourceLiteral);
resourceLiteral.AnalyzeNode();
return;
}
else
{
// error! must be a constant
Context.HandleError(
JSError.ResourceReferenceMustBeConstant,
true);
}
}
else
{
// error! can only be a single constant argument to the string resource object.
// the parser will only have zero or one arguments, so this must be zero
// (since the parser won't pass multiple args to a [] operator)
Context.HandleError(
JSError.ResourceReferenceMustBeConstant,
true);
}
}
}
// and finally, if this is a backets call and the argument is a constantwrapper that can
// be an identifier, just change us to a member node: obj["prop"] to obj.prop.
// but ONLY if the string value is "safe" to be an identifier. Even though the ECMA-262
// spec says certain Unicode categories are okay, in practice the various major browsers
// all seem to have problems with certain characters in identifiers. Rather than risking
// some browsers breaking when we change this syntax, don't do it for those "danger" categories.
if (m_inBrackets && m_args != null)
{
// see if there is a single, constant argument
string argText = m_args.SingleConstantArgument;
if (argText != null)
{
// see if we want to replace the name
string newName;
if (Parser.Settings.HasRenamePairs && Parser.Settings.ManualRenamesProperties &&
Parser.Settings.IsModificationAllowed(TreeModifications.PropertyRenaming) &&
!string.IsNullOrEmpty(newName = Parser.Settings.GetNewName(argText)))
{
// yes -- we are going to replace the name, either as a string literal, or by converting
// to a member-dot operation.
// See if we can't turn it into a dot-operator. If we can't, then we just want to replace the operator with
// a new constant wrapper. Otherwise we'll just replace the operator with a new constant wrapper.
if (Parser.Settings.IsModificationAllowed(TreeModifications.BracketMemberToDotMember) &&
JSScanner.IsSafeIdentifier(newName) &&
!JSScanner.IsKeyword(newName))
{
// the new name is safe to convert to a member-dot operator.
// but we don't want to convert the node to the NEW name, because we still need to Analyze the
// new member node -- and it might convert the new name to something else. So instead we're
// just going to convert this existing string to a member node WITH THE OLD STRING,
// and THEN analyze it (which will convert the old string to newName)
Member replacementMember = new Member(Context, Parser, m_func, argText);
Parent.ReplaceChild(this, replacementMember);
// this analyze call will convert the old-name member to the newName value
replacementMember.AnalyzeNode();
return;
}
else
{
// nope; can't convert to a dot-operator.
// we're just going to replace the first argument with a new string literal
// and continue along our merry way.
m_args.ReplaceChild(m_args[0], new ConstantWrapper(newName, PrimitiveType.String, m_args[0].Context, Parser));
}
}
else if (Parser.Settings.IsModificationAllowed(TreeModifications.BracketMemberToDotMember) &&
JSScanner.IsSafeIdentifier(argText) &&
!JSScanner.IsKeyword(argText))
{
// not a replacement, but the string literal is a safe identifier. So we will
// replace this call node with a Member-dot operation
Member replacementMember = new Member(Context, Parser, m_func, argText);
Parent.ReplaceChild(this, replacementMember);
replacementMember.AnalyzeNode();
return;
}
}
}
// call the base class to recurse
base.AnalyzeNode();
// call this AFTER recursing to give the fields a chance to resolve, because we only
// want to make this replacement if we are working on the global Date object.
if (!m_inBrackets && !m_isConstructor &&
(m_args == null || m_args.Count == 0) &&
member != null && string.CompareOrdinal(member.Name, "getTime") == 0 &&
Parser.Settings.IsModificationAllowed(TreeModifications.DateGetTimeToUnaryPlus))
{
// this is not a constructor and it's not a brackets call, and there are no arguments.
// if the function is a member operation to "getTime" and the object of the member is a
// constructor call to the global "Date" object (not a local), then we want to replace the call
// with a unary plus on the Date constructor. Converting to numeric type is the same as
// calling getTime, so it's the equivalent with much fewer bytes.
CallNode dateConstructor = member.Root as CallNode;
if (dateConstructor != null &&
dateConstructor.IsConstructor)
{
// lookup for the predifined (not local) "Date" field
Lookup lookup = dateConstructor.Function as Lookup;
if (lookup != null && string.CompareOrdinal(lookup.Name, "Date") == 0 &&
lookup.LocalField == null)
{
// this is in the pattern: (new Date()).getTime()
// we want to replace it with +new Date
// use the same date constructor node as the operand
NumericUnary unary = new NumericUnary(Context, Parser, dateConstructor, JSToken.Plus);
// replace us (the call to the getTime method) with this unary operator
Parent.ReplaceChild(this, unary);
// don't need to AnalyzeNode on the unary operator. The operand has already
// been analyzed when we recursed, and the unary operator wouldn't do anything
// special anyway (since the operand is not a numeric constant)
}
}
}
else if (Parser.Settings.EvalTreatment != EvalTreatment.Ignore)
{
// if this is a window.eval call, then we need to mark this scope as unknown just as
// we would if this was a regular eval call.
// (unless, of course, the parser settings say evals are safe)
// call AFTER recursing so we know the left-hand side properties have had a chance to
// lookup their fields to see if they are local or global
if (member != null && string.CompareOrdinal(member.Name, "eval") == 0)
{
if (member.LeftHandSide.IsWindowLookup)
{
// this is a call to window.eval()
// mark this scope as unknown so we don't crunch out locals
// we might reference in the eval at runtime
ScopeStack.Peek().IsKnownAtCompileTime = false;
}
}
else
{
CallNode callNode = m_func as CallNode;
if (callNode != null &&
callNode.InBrackets &&
callNode.LeftHandSide.IsWindowLookup &&
callNode.Arguments.IsSingleConstantArgument("eval"))
{
// this is a call to window["eval"]
// mark this scope as unknown so we don't crunch out locals
// we might reference in the eval at runtime
ScopeStack.Peek().IsKnownAtCompileTime = false;
}
}
}
/* REVIEW: may be too late. lookups may alread have been analyzed and
* found undefined
* // check to see if this is an assignment to a window["prop"] structure
* BinaryOperator binaryOp = Parent as BinaryOperator;
* if (binaryOp != null && binaryOp.IsAssign
* && m_inBrackets
* && m_func.IsWindowLookup
* && m_args != null)
* {
* // and IF the property is a non-empty constant that isn't currently
* // a global field...
* string propertyName = m_args.SingleConstantArgument;
* if (!string.IsNullOrEmpty(propertyName)
* && Parser.GlobalScope[propertyName] == null)
* {
* // we want to also add it to the global fields so it's not undefined
* Parser.GlobalScope.DeclareField(propertyName, null, 0);
* }
* }
*/
}
private JSScanner GetJSScanner() {
if (_scanner == null) {
_scanner = new JSScanner(new CodeSettings() { AllowShebangLine = true });
}
return _scanner;
}
private LineTokenization TokenizeLine(JSScanner JSScanner, ITextSnapshot snapshot, object previousLineState, int lineNo) {
ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo);
SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak);
var tcp = new SnapshotSpanSourceCodeReader(lineSpan);
JSScanner.Initialize(
lineSpan.GetText(),
previousLineState,
new SourceLocation(0, lineNo + 1, 1)
);
try {
var tokens = JSScanner.ReadTokens(lineSpan.Length).Select(ToTokenKind).ToArray();
return new LineTokenization(tokens, JSScanner.CurrentState);
} finally {
JSScanner.Uninitialize();
}
}
/// <summary>
/// Rescans the part of the buffer affected by a change.
/// Scans a contiguous sub-<paramref name="span"/> of a larger code span which starts at <paramref name="codeStartLine"/>.
/// </summary>
private void ApplyChange(JSScanner JSScanner, ITextSnapshot snapshot, Span span) {
int firstLine = snapshot.GetLineNumberFromPosition(span.Start);
int lastLine = snapshot.GetLineNumberFromPosition(span.Length > 0 ? span.End - 1 : span.End);
Contract.Assert(firstLine >= 0);
// find the closest line preceding firstLine for which we know categorizer state, stop at the codeStartLine:
LineTokenization lineTokenization;
firstLine = _tokenCache.IndexOfPreviousTokenization(firstLine, 0, out lineTokenization) + 1;
object state = lineTokenization.State;
int currentLine = firstLine;
object previousState;
while (currentLine < snapshot.LineCount) {
previousState = _tokenCache.TryGetTokenization(currentLine, out lineTokenization) ? lineTokenization.State : null;
_tokenCache[currentLine] = lineTokenization = TokenizeLine(JSScanner, snapshot, state, currentLine);
state = lineTokenization.State;
// stop if we visted all affected lines and the current line has no tokenization state or its previous state is the same as the new state:
if (currentLine > lastLine && (previousState == null || previousState.Equals(state))) {
break;
}
currentLine++;
}
// classification spans might have changed between the start of the first and end of the last visited line:
int changeStart = snapshot.GetLineFromLineNumber(firstLine).Start;
int changeEnd = (currentLine < snapshot.LineCount) ? snapshot.GetLineFromLineNumber(currentLine).End : snapshot.Length;
if (changeStart < changeEnd) {
var classificationChanged = ClassificationChanged;
if (classificationChanged != null) {
var args = new ClassificationChangedEventArgs(new SnapshotSpan(snapshot, new Span(changeStart, changeEnd - changeStart)));
classificationChanged(this, args);
}
}
}
