private JSKeyword(JSToken token, string name, JSKeyword next) { m_name = name; m_token = token; m_length = m_name.Length; m_next = next; }
protected UnaryOperator(Context context, JSParser parser, AstNode operand, JSToken operatorToken) : base(context, parser) { Operand = operand; OperatorToken = operatorToken; if (Operand != null) Operand.Parent = this; }
private JSKeyword(JSToken token, string name, JSKeyword next) { this.name = name; this.next = next; this.token = token; this.length = this.name.Length; }
internal TokenWithSpan(LocationResolver indexResolver, int start, int end, JSToken token) { _token = token; _start = start; _end = end; _indexResolver = indexResolver; }
private static Object DoOp(ulong x, ulong y, JSToken operatorTok) { switch (operatorTok) { case JSToken.Divide: return(((double)x) / (double)y); case JSToken.Minus: try{ checked { return(x - y); } }catch (OverflowException) { return(((double)x) - (double)y); } case JSToken.Modulo: if (y == 0) { return(Double.NaN); } return(x % y); case JSToken.Multiply: try{ checked { return(x * y); } }catch (OverflowException) { return(((double)x) * (double)y); } default: throw new JScriptException(JSError.InternalError); } }
public TokenWithSpan(TokenWithSpan currentToken, JSToken newToken) { _indexResolver = currentToken._indexResolver; _start = currentToken._start; _end = currentToken._end; _token = newToken; }
private static Object DoOp(long x, long y, JSToken operatorTok) { switch (operatorTok) { case JSToken.Divide: return(((double)x) / (double)y); case JSToken.Minus: try{ checked { return(x - y); } }catch (OverflowException) { return(((double)x) - (double)y); } case JSToken.Modulo: if (y == 0) { return(Double.NaN); } long result = x % y; if (result == 0) //Need to result in a signed 0 { if (x < 0) { if (y < 0) { return(0); } else { return(1.0 / Double.NegativeInfinity); } } else if (y < 0) { return(1.0 / Double.NegativeInfinity); } else { return(0); } } return(result); case JSToken.Multiply: if (x == 0 || y == 0) { return(((double)x) * (double)y); //Need to result in a signed 0 } try{ checked { return(x * y); } }catch (OverflowException) { return(((double)x) * (double)y); } default: throw new JScriptException(JSError.InternalError); } }
private static Object DoOp(ulong x, ulong y, JSToken operatorTok) { switch (operatorTok) { case JSToken.Divide: return(((double)x) / (double)y); case JSToken.Minus: ulong r = x - y; if (r <= x) // Since y is 0 or positive, if the result left of x or at x, no overflow occured. { return(r); } return(((double)x) - (double)y); case JSToken.Modulo: if (y == 0) { return(Double.NaN); } return(x % y); case JSToken.Multiply: try{ checked { return(x * y); } }catch (OverflowException) { return(((double)x) * (double)y); } default: throw new JScriptException(JSError.InternalError); } }
// Evaluate a numeric binary operator on two values. public static Object DoOp(Object v1, Object v2, JSToken operatorTok) { double n1 = Convert.ToNumber(v1); double n2 = Convert.ToNumber(v2); switch (operatorTok) { case JSToken.Minus: { return(n1 - n2); } // Not reached. case JSToken.Multiply: { return(n1 * n2); } // Not reached. case JSToken.Divide: { return(n1 / n2); } // Not reached. case JSToken.Modulo: { return(n1 % n2); } // Not reached. } throw new JScriptException(JSError.InternalError); }
internal static TokenColor ColorFromToken(Context context) { JSToken token = context.GetToken(); if (JSScanner.IsKeyword(token)) { return(TokenColor.COLOR_KEYWORD); } if (JSToken.Identifier == token) { if (context.Equals("eval")) { return(TokenColor.COLOR_KEYWORD); } return(TokenColor.COLOR_IDENTIFIER); } if (JSToken.StringLiteral == token) { return(TokenColor.COLOR_STRING); } if ((JSToken.NumericLiteral == token) || (JSToken.IntegerLiteral == token)) { return(TokenColor.COLOR_NUMBER); } if ((JSToken.Comment == token) || (JSToken.UnterminatedComment == token)) { return(TokenColor.COLOR_COMMENT); } if (JSScanner.IsOperator(token)) { return(TokenColor.COLOR_OPERATOR); } return(TokenColor.COLOR_TEXT); }
internal BinaryOp (AST parent, AST left, AST right, JSToken op, Location location) : base (parent, location) { operand1 = left; operand2 = right; operatorTok = op; }
public virtual SourceState GetStateForText(string sourceCode, SourceState state) { if (sourceCode != null) { this._state = SourceState.STATE_COLOR_NORMAL; Context sourceContext = new Context(null, sourceCode); this._scanner.SetSource(sourceContext); if ((SourceState.STATE_COLOR_COMMENT == state) && (this._scanner.SkipMultiLineComment() > sourceCode.Length)) { this._state = SourceState.STATE_COLOR_COMMENT; return(this._state); } this._scanner.GetNextToken(); JSToken none = JSToken.None; while (sourceContext.GetToken() != JSToken.EndOfFile) { none = sourceContext.GetToken(); this._scanner.GetNextToken(); } if (JSToken.UnterminatedComment == none) { this._state = SourceState.STATE_COLOR_COMMENT; } } return(this._state); }
internal static Object DoOp(int i, int j, JSToken operatorTok) { switch (operatorTok) { case JSToken.BitwiseAnd: return(i & j); case JSToken.BitwiseOr: return(i | j); case JSToken.BitwiseXor: return(i ^ j); case JSToken.LeftShift: return(i << j); case JSToken.RightShift: return(i >> j); case JSToken.UnsignedRightShift: return(((uint)i) >> j); default: throw new JScriptException(JSError.InternalError); } }
// Evaluate a numeric binary operator on two values. public static Object DoOp(Object v1, Object v2, JSToken operatorTok) { double n1 = Convert.ToNumber(v1); double n2 = Convert.ToNumber(v2); switch(operatorTok) { case JSToken.Minus: { return (n1 - n2); } // Not reached. case JSToken.Multiply: { return (n1 * n2); } // Not reached. case JSToken.Divide: { return (n1 / n2); } // Not reached. case JSToken.Modulo: { return (n1 % n2); } // Not reached. } throw new JScriptException(JSError.InternalError); }
internal BinaryOp(AST parent, AST left, AST right, JSToken op, Location location) : base(parent, location) { operand1 = left; operand2 = right; operatorTok = op; }
internal PostOrPrefixOperator(AST parent, AST operand, JSToken oper, bool prefix, Location location) : base(parent, location) { this.operand = operand; this.oper = oper; this.prefix = prefix; }
/*internal bool Exists(string target) { JSKeyword keyword = this; while (keyword != null) { if (keyword.m_name == target) { return true; } keyword = keyword.m_next; } return false; }*/ internal static string CanBeIdentifier(JSToken keyword) { switch (keyword) { // always allowed case JSToken.Get: return "get"; case JSToken.Set: return "set"; // not in strict mode case JSToken.Implements: return "implements"; case JSToken.Interface: return "interface"; case JSToken.Let: return "let"; case JSToken.Package: return "package"; case JSToken.Private: return "private"; case JSToken.Protected: return "protected"; case JSToken.Public: return "public"; case JSToken.Static: return "static"; case JSToken.Yield: return "yield"; // apparently never allowed for Chrome, so we want to treat it // differently, too case JSToken.Native: return "native"; // no other tokens can be identifiers default: return null; } }
internal NumericUnary(Context context, AST operand, JSToken operatorTok) : base(context, operand) { this.operatorTok = operatorTok; this.operatorMeth = null; this.type = null; }
//public Dictionary<JSToken, int> TokenCounts; public JSScanner(Context sourceContext) { m_keywords = s_Keywords; m_previousToken = JSToken.None; EatUnnecessaryCCOn = true; UsePreprocessorDefines = true; SetSource(sourceContext); }
internal BinaryOp(Context context, AST operand1, AST operand2, JSToken operatorTok) : base(context) { this.operand1 = operand1; this.operand2 = operand2; this.operatorTok = operatorTok; this.type1 = null; this.type2 = null; this.operatorMeth = null; }
private static object DoOp(long x, long y, JSToken operatorTok) { switch (operatorTok) { case JSToken.Multiply: if ((x != 0L) && (y != 0L)) { try { return(x * y); } catch (OverflowException) { return(x * y); } } return(x * y); case JSToken.Divide: return(((double)x) / ((double)y)); case JSToken.Modulo: if (y != 0L) { long num2 = x % y; if (num2 != 0L) { return(num2); } if (x < 0L) { if (y < 0L) { return(0); } return(0.0); } if (y < 0L) { return(0.0); } return(0); } return((double)1.0 / (double)0.0); case JSToken.Minus: { long num = x - y; if ((num < x) == (y > 0L)) { return(num); } return(x - y); } } throw new JScriptException(JSError.InternalError); }
private static object DoOp(long x, long y, JSToken operatorTok) { switch (operatorTok) { case JSToken.Multiply: if ((x != 0L) && (y != 0L)) { try { return (x * y); } catch (OverflowException) { return (x * y); } } return (x * y); case JSToken.Divide: return (((double) x) / ((double) y)); case JSToken.Modulo: if (y != 0L) { long num2 = x % y; if (num2 != 0L) { return num2; } if (x < 0L) { if (y < 0L) { return 0; } return 0.0; } if (y < 0L) { return 0.0; } return 0; } return (double) 1.0 / (double) 0.0; case JSToken.Minus: { long num = x - y; if ((num < x) == (y > 0L)) { return num; } return (x - y); } } throw new JScriptException(JSError.InternalError); }
protected UnaryOperator(Context context, JSParser parser, AstNode operand, JSToken operatorToken) : base(context, parser) { m_operand = operand; m_operatorToken = operatorToken; if (m_operand != null) { m_operand.Parent = this; } }
// Constructors. internal Context(String source) { startPosition = 0; endPosition = source.Length; startLine = 1; startLinePosition = 0; endLine = 1; endLinePosition = 0; token = JSToken.None; this.source = source; }
internal static Type Operand2Type(JSToken operatorTok, Type bbrType) { switch (operatorTok) { case JSToken.LeftShift: case JSToken.RightShift: case JSToken.UnsignedRightShift: return(Typeob.Int32); } return(bbrType); }
public Context(DocumentContext document, int startLineNumber, int startLinePosition, int startPosition, int endLineNumber, int endLinePosition, int endPosition, JSToken token) : this(document) { StartLineNumber = startLineNumber; StartLinePosition = startLinePosition; StartPosition = startPosition; EndLineNumber = endLineNumber; EndLinePosition = endLinePosition; EndPosition = endPosition; Token = token; }
internal static void EmitRelationalComp(ILGenerator ig, Relational re) { JSToken op = re.op; if (op == JSToken.Instanceof) { return; } else if (op == JSToken.In) { ig.Emit(OpCodes.Box, typeof(bool)); return; } Label true_case = ig.DefineLabel(); Label box_to_bool = ig.DefineLabel(); ig.Emit(OpCodes.Ldc_I4_0); ig.Emit(OpCodes.Conv_R8); OpCode opcode; switch (op) { case JSToken.LessThan: opcode = OpCodes.Blt; break; case JSToken.LessThanEqual: opcode = OpCodes.Ble; break; case JSToken.GreaterThan: opcode = OpCodes.Bgt; break; case JSToken.GreaterThanEqual: opcode = OpCodes.Bge; break; default: Console.WriteLine(re.Location.LineNumber); throw new NotImplementedException(); } ig.Emit(opcode, true_case); ig.Emit(OpCodes.Ldc_I4_0); ig.Emit(OpCodes.Br, box_to_bool); ig.MarkLabel(true_case); ig.Emit(OpCodes.Ldc_I4_1); ig.MarkLabel(box_to_bool); ig.Emit(OpCodes.Box, typeof(bool)); }
internal Context(DocumentContext document, String source_string){ this.document = document; this.source_string = source_string; this.lineNumber = 1; this.startLinePos = 0; this.startPos = 0; this.endLineNumber = 1; this.endLinePos = 0; this.endPos = (source_string == null) ? -1 : source_string.Length; this.token = JSToken.None; this.errorReported = 1000000; }
internal Context(DocumentContext document, string source_string, int lineNumber, int startLinePos, int startPos, int endLineNumber, int endLinePos, int endPos, JSToken token) { this.document = document; this.source_string = source_string; this.lineNumber = lineNumber; this.startLinePos = startLinePos; this.startPos = startPos; this.endLineNumber = endLineNumber; this.endLinePos = endLinePos; this.endPos = endPos; this.token = token; this.errorReported = 0xf4240; }
internal Context(DocumentContext document, string source_string) { this.document = document; this.source_string = source_string; this.lineNumber = 1; this.startLinePos = 0; this.startPos = 0; this.endLineNumber = 1; this.endLinePos = 0; this.endPos = (source_string == null) ? -1 : source_string.Length; this.token = JSToken.None; this.errorReported = 0xf4240; }
public Context(DocumentContext document, int lineNumber, int startLinePos, int startPos, int endLineNumber, int endLinePos, int endPos, JSToken token) { Document = document; StartLineNumber = lineNumber; StartLinePosition = startLinePos; StartPosition = startPos; EndLineNumber = endLineNumber; EndLinePosition = endLinePos; EndPosition = endPos; Token = token; m_errorReported = 1000000; }
// Parse a number. private void TestNum(String value, JSToken expected) { JSScanner scanner; Context token; scanner = JSScannerTest.TestCreateScanner(value); token = JSScannerTest.TestGetTokenContext(scanner); scanner.GetNextToken(); AssertEquals("TestNum[" + value + "] (1)", value, token.GetCode()); AssertEquals("TestNum[" + value + "] (2)", expected, token.GetToken()); }
// Constructor. public JSParser(Context context) { this.context = context; if(context != null) { scanner = new JSScanner(context); } else { scanner = new JSScanner(); } tokenInfo = scanner.GetTokenContext(); token = tokenInfo.token; numErrors = 0; }
public Context(DocumentContext document, string sourceCode, int lineNumber, int startLinePos, int startPos, int endLineNumber, int endLinePos, int endPos, JSToken token, string fileContext) { Document = document; SourceString = sourceCode; StartLineNumber = lineNumber; StartLinePosition = startLinePos; StartPosition = startPos; EndLineNumber = endLineNumber; EndLinePosition = endLinePos; EndPosition = endPos; Token = token; m_errorReported = 1000000; FileContext = fileContext; }
public Context(DocumentContext document) { if (document == null) { throw new ArgumentNullException("document"); } Document = document; StartLineNumber = 1; EndLineNumber = 1; EndPosition = Document.Source.IfNotNull(s => s.Length); Token = JSToken.None; }
/*internal bool Exists(string target) * { * JSKeyword keyword = this; * while (keyword != null) * { * if (keyword.m_name == target) * { * return true; * } * keyword = keyword.m_next; * } * return false; * }*/ internal static string CanBeIdentifier(JSToken keyword) { switch (keyword) { // always allowed case JSToken.Get: return("get"); case JSToken.Set: return("set"); case JSToken.Super: return("super"); case JSToken.Of: return("of"); // what about EcmaScript 6? Does this become a reserved word? case JSToken.Module: return("module"); // not in strict mode case JSToken.Implements: return("implements"); case JSToken.Interface: return("interface"); case JSToken.Let: return("let"); case JSToken.Package: return("package"); case JSToken.Private: return("private"); case JSToken.Protected: return("protected"); case JSToken.Public: return("public"); case JSToken.Async: return("async"); case JSToken.Await: return("await"); case JSToken.Static: return("static"); case JSToken.Yield: return("yield"); // apparently never allowed for Chrome, so we want to treat it // differently, too case JSToken.Native: return("native"); // no other tokens can be identifiers default: return(null); } }
public JSToken PeekToken() { JSToken jSToken; if (this.GetToken()) { jSToken = this.Token; this.UngetToken(); } else { jSToken = new JSToken(); jSToken.Value = string.Empty; } return(jSToken); }
internal bool HasToken(JSToken token) { for (TokenSetListItem item = this._tokenSet; item != null; item = item._next) { int index = 0; int length = item._tokens.Length; while (index < length) { if (item._tokens[index] == token) { return(true); } index++; } } return(false); }
internal bool HasToken(JSToken token) { for (TokenSetListItem item = this._tokenSet; item != null; item = item._next) { int index = 0; int length = item._tokens.Length; while (index < length) { if (item._tokens[index] == token) { return true; } index++; } } return false; }
private static bool NeedsParens(AstNode node, JSToken refToken) { bool needsParens = false; // assignments and commas are the only operators that need parens // around them. Conditional is pretty low down the list BinaryOperator binaryOp = node as BinaryOperator; if (binaryOp != null) { OpPrec thisPrecedence = JSScanner.GetOperatorPrecedence(refToken); OpPrec nodePrecedence = JSScanner.GetOperatorPrecedence(binaryOp.OperatorToken); needsParens = (nodePrecedence < thisPrecedence); } return(needsParens); }
internal static string CanBeIdentifier(JSToken keyword){ switch (keyword){ case JSToken.Abstract: return "abstract"; case JSToken.Assert: return "assert"; case JSToken.Boolean: return "boolean"; case JSToken.Byte: return "byte"; case JSToken.Char: return "char"; case JSToken.Decimal: return "decimal"; case JSToken.Double: return "double"; case JSToken.Ensure: return "ensure"; case JSToken.Enum: return "enum"; case JSToken.Event: return "event"; case JSToken.Final: return "final"; case JSToken.Float: return "float"; case JSToken.Get: return "get"; case JSToken.Goto: return "goto"; case JSToken.Implements: return "implements"; case JSToken.Int: return "int"; case JSToken.Interface: return "interface"; case JSToken.Internal: return "internal"; case JSToken.Invariant: return "invariant"; case JSToken.Long: return "long"; case JSToken.Namespace: return "namespace"; case JSToken.Native: return "native"; case JSToken.Package: return "package"; case JSToken.Private: return "private"; case JSToken.Protected: return "protected"; case JSToken.Public: return "public"; case JSToken.Require: return "require"; case JSToken.Sbyte: return "sbyte"; case JSToken.Set: return "set"; case JSToken.Short: return "short"; case JSToken.Static: return "static"; case JSToken.Synchronized: return "synchronized"; case JSToken.Throws: return "throws"; case JSToken.Transient: return "transient"; case JSToken.Void: return "void"; case JSToken.Volatile: return "volatile"; case JSToken.Uint : return "uint"; case JSToken.Ulong : return "ulong"; case JSToken.Ushort : return "ushort"; case JSToken.Use : return "use"; default: return null; } }
private static object DoOp(double x, double y, JSToken operatorTok) { switch (operatorTok) { case JSToken.Multiply: return (x * y); case JSToken.Divide: return (x / y); case JSToken.Modulo: return (x % y); case JSToken.Minus: return (x - y); } throw new JScriptException(JSError.InternalError); }
internal void Remove(JSToken[] tokens) { TokenSetListItem item = this._tokenSet; TokenSetListItem item2 = null; while (item != null) { if (item._tokens == tokens) { if (item2 == null) { this._tokenSet = this._tokenSet._next; return; } item2._next = item._next; return; } item2 = item; item = item._next; } }
private static object DoOp(int x, int y, JSToken operatorTok) { switch (operatorTok) { case JSToken.Multiply: if ((x != 0) && (y != 0)) { try { return (x * y); } catch (OverflowException) { return (x * y); } } return (x * y); case JSToken.Divide: return (((double) x) / ((double) y)); case JSToken.Modulo: if ((x > 0) && (y > 0)) { return (x % y); } return (((double) x) % ((double) y)); case JSToken.Minus: { int num = x - y; if ((num < x) == (y > 0)) { return num; } return (x - y); } } throw new JScriptException(JSError.InternalError); }
internal static object DoOp(int i, int j, JSToken operatorTok) { switch (operatorTok) { case JSToken.BitwiseOr: return (i | j); case JSToken.BitwiseXor: return (i ^ j); case JSToken.BitwiseAnd: return (i & j); case JSToken.LeftShift: return (i << j); case JSToken.RightShift: return (i >> j); case JSToken.UnsignedRightShift: return (uint) (i >> j); } throw new JScriptException(JSError.InternalError); }
internal NumericBinaryAssign(Context context, AST operand1, AST operand2, JSToken operatorTok) : base(context, operand1, operand2, operatorTok){ this.binOp = new NumericBinary(context, operand1, operand2, operatorTok); this.metaData = null; }
internal Relational(Context context, AST operand1, AST operand2, JSToken operatorTok) : base(context, operand1, operand2, operatorTok) { }
public void GetNextToken() { JSToken token = JSToken.None; GotEndOfLine = false; m_importantComments = null; try { int thisCurrentLine = CurrentLine; nextToken: // skip any blanks, setting a state flag if we find any bool ws = JSScanner.IsBlankSpace(GetChar(m_currentPos)); if (ws && !RawTokens) { // we're not looking for war tokens, so just want to eat the whitespace while (JSScanner.IsBlankSpace(GetChar(++m_currentPos))) ; } m_currentToken.StartPosition = m_startPos = m_currentPos; m_currentToken.StartLineNumber = CurrentLine; m_currentToken.StartLinePosition = StartLinePosition; char c = GetChar(m_currentPos++); switch (c) { case (char)0: if (m_currentPos >= m_endPos) { m_currentPos--; token = JSToken.EndOfFile; if (m_ccIfLevel > 0) { m_currentToken.EndLineNumber = CurrentLine; m_currentToken.EndLinePosition = StartLinePosition; m_currentToken.EndPosition = m_currentPos; HandleError(JSError.NoCCEnd); } break; } if (RawTokens) { // if we are just looking for raw tokens, return this one as an error token token = JSToken.Error; break; } else { // otherwise eat it goto nextToken; } case '=': token = JSToken.Assign; if ('=' == GetChar(m_currentPos)) { m_currentPos++; token = JSToken.Equal; if ('=' == GetChar(m_currentPos)) { m_currentPos++; token = JSToken.StrictEqual; } } break; case '>': token = JSToken.GreaterThan; if ('>' == GetChar(m_currentPos)) { m_currentPos++; token = JSToken.RightShift; if ('>' == GetChar(m_currentPos)) { m_currentPos++; token = JSToken.UnsignedRightShift; } } if ('=' == GetChar(m_currentPos)) { m_currentPos++; token = token == JSToken.GreaterThan ? JSToken.GreaterThanEqual : token == JSToken.RightShift ? JSToken.RightShiftAssign : token == JSToken.UnsignedRightShift ? JSToken.UnsignedRightShiftAssign : token; } break; case '<': if (AllowEmbeddedAspNetBlocks && '%' == GetChar(m_currentPos)) { token = JSToken.AspNetBlock; ScanAspNetBlock(); } else { token = JSToken.LessThan; if ('<' == GetChar(m_currentPos)) { m_currentPos++; token = JSToken.LeftShift; } if ('=' == GetChar(m_currentPos)) { m_currentPos++; if (token == JSToken.LessThan) { token = JSToken.LessThanEqual; } else { token = JSToken.LeftShiftAssign; } } } break; case '!': token = JSToken.LogicalNot; if ('=' == GetChar(m_currentPos)) { m_currentPos++; token = JSToken.NotEqual; if ('=' == GetChar(m_currentPos)) { m_currentPos++; token = JSToken.StrictNotEqual; } } break; case ',': token = JSToken.Comma; break; case '~': token = JSToken.BitwiseNot; break; case '?': token = JSToken.ConditionalIf; break; case ':': token = JSToken.Colon; break; case '.': token = JSToken.AccessField; c = GetChar(m_currentPos); if (JSScanner.IsDigit(c)) { token = ScanNumber('.'); } break; case '&': token = JSToken.BitwiseAnd; c = GetChar(m_currentPos); if ('&' == c) { m_currentPos++; token = JSToken.LogicalAnd; } else if ('=' == c) { m_currentPos++; token = JSToken.BitwiseAndAssign; } break; case '|': token = JSToken.BitwiseOr; c = GetChar(m_currentPos); if ('|' == c) { m_currentPos++; token = JSToken.LogicalOr; } else if ('=' == c) { m_currentPos++; token = JSToken.BitwiseOrAssign; } break; case '+': token = JSToken.Plus; c = GetChar(m_currentPos); if ('+' == c) { m_currentPos++; token = JSToken.Increment; } else if ('=' == c) { m_currentPos++; token = JSToken.PlusAssign; } break; case '-': token = JSToken.Minus; c = GetChar(m_currentPos); if ('-' == c) { m_currentPos++; token = JSToken.Decrement; } else if ('=' == c) { m_currentPos++; token = JSToken.MinusAssign; } break; case '*': token = JSToken.Multiply; if ('=' == GetChar(m_currentPos)) { m_currentPos++; token = JSToken.MultiplyAssign; } break; case '\\': m_currentPos--; if (IsIdentifierStartChar(ref c)) { m_currentPos++; ScanIdentifier(); token = JSToken.Identifier; break; } m_currentPos++; // move on c = GetChar(m_currentPos); if ('a' <= c && c <= 'z') { JSKeyword keyword = m_keywords[c - 'a']; if (null != keyword) { m_currentToken.StartPosition++; token = ScanKeyword(keyword); m_currentToken.StartPosition--; if (token != JSToken.Identifier) { token = JSToken.Identifier; break; } } } m_currentPos = m_currentToken.StartPosition + 1; HandleError(JSError.IllegalChar); break; case '/': token = JSToken.Divide; c = GetChar(m_currentPos); switch (c) { case '/': m_inSingleLineComment = true; c = GetChar(++m_currentPos); // see if there is a THIRD slash character if (c == '/') { // advance past the slash ++m_currentPos; // check for some AjaxMin preprocessor comments if (CheckSubstring(m_currentPos, "#DEBUG")) { if (SkipDebugBlocks) { // skip until we hit ///#ENDDEBUG, but only if we are stripping debug statements PPSkipToDirective("#ENDDEBUG"); // if we are asking for raw tokens, we DON'T want to return these comments or the code // they stripped away. if (RawTokens) { SkipSingleLineComment(); goto nextToken; } } } else if (UsePreprocessorDefines) { if (CheckSubstring(m_currentPos, "#IFDEF")) { // skip past the token and any blanks m_currentPos += 6; SkipBlanks(); // if we encountered a line-break here, then ignore this directive if (!GotEndOfLine) { // get an identifier from the input var identifier = PPScanIdentifier(); if (!string.IsNullOrEmpty(identifier)) { // set a state so that if we hit an #ELSE directive, we skip to #ENDIF m_inIfDefDirective = true; // if there is a dictionary AND the identifier is in it... if (m_defines == null || !m_defines.ContainsKey(identifier)) { // it's NOT defined! // skip to #ELSE or #ENDIF and continue processing normally. if (PPSkipToDirective("#ENDIF", "#ELSE") == 0) { // encountered the #ENDIF directive, so we know to reset the flag m_inIfDefDirective = false; } } // if we are asking for raw tokens, we DON'T want to return these comments or the code // they may have stripped away. if (RawTokens) { SkipSingleLineComment(); goto nextToken; } } } } else if (CheckSubstring(m_currentPos, "#ELSE") && m_inIfDefDirective) { // reset the state that says we were in an #IFDEF construct m_inIfDefDirective = false; // ...then we now want to skip until the #ENDIF directive PPSkipToDirective("#ENDIF"); // if we are asking for raw tokens, we DON'T want to return these comments or the code // they stripped away. if (RawTokens) { SkipSingleLineComment(); goto nextToken; } } else if (CheckSubstring(m_currentPos, "#ENDIF") && m_inIfDefDirective) { // reset the state that says we were in an #IFDEF construct m_inIfDefDirective = false; // if we are asking for raw tokens, we DON'T want to return this comment. if (RawTokens) { SkipSingleLineComment(); goto nextToken; } } else if (CheckSubstring(m_currentPos, "#DEFINE")) { // skip past the token and any blanks m_currentPos += 7; SkipBlanks(); // if we encountered a line-break here, then ignore this directive if (!GotEndOfLine) { // get an identifier from the input var identifier = PPScanIdentifier(); if (!string.IsNullOrEmpty(identifier)) { // if there is no dictionary of defines yet, create one now if (m_defines == null) { m_defines = new Dictionary<string, string>(); } // if the identifier is not already in the dictionary, add it now if (!m_defines.ContainsKey(identifier)) { m_defines.Add(identifier, identifier); } // if we are asking for raw tokens, we DON'T want to return this comment. if (RawTokens) { SkipSingleLineComment(); goto nextToken; } } } } else if (CheckSubstring(m_currentPos, "#UNDEF")) { // skip past the token and any blanks m_currentPos += 6; SkipBlanks(); // if we encountered a line-break here, then ignore this directive if (!GotEndOfLine) { // get an identifier from the input var identifier = PPScanIdentifier(); // if there was an identifier and we have a dictionary of "defines" and the // identifier is in that dictionary... if (!string.IsNullOrEmpty(identifier)) { if (m_defines != null && m_defines.ContainsKey(identifier)) { // remove the identifier from the "defines" dictionary m_defines.Remove(identifier); } // if we are asking for raw tokens, we DON'T want to return this comment. if (RawTokens) { SkipSingleLineComment(); goto nextToken; } } } } } } else if (!RawTokens && c == '@' && !IgnoreConditionalCompilation && !m_peekModeOn) { // we got //@ // if we have not turned on conditional-compilation yet, then check to see if that's // what we're trying to do now if (!m_preProcessorOn) { // we are currently on the @ -- start peeking from there if (!CheckSubstring(m_currentPos + 1, "cc_on")) { // we aren't turning on conditional comments. We need to ignore this comment // as just another single-line comment SkipSingleLineComment(); goto nextToken; } } // if the NEXT character is not an identifier character, then we need to skip // the @ character -- otherwise leave it there if (!IsValidIdentifierStart(GetChar(m_currentPos + 1))) { ++m_currentPos; } // if we aren't already in a conditional comment if (!m_inConditionalComment) { // we are now m_inConditionalComment = true; token = JSToken.ConditionalCommentStart; break; } // already in conditional comment, so just ignore the start of a new // conditional comment. it's superfluous. goto nextToken; } SkipSingleLineComment(); if (RawTokens) { // raw tokens -- just return the comment token = JSToken.Comment; break; } else { // if we're still in a multiple-line comment, then we must've been in // a multi-line CONDITIONAL comment, in which case this normal one-line comment // won't turn off conditional comments just because we hit the end of line. if (!m_inMultipleLineComment && m_inConditionalComment) { m_inConditionalComment = false; token = JSToken.ConditionalCommentEnd; break; } goto nextToken; // read another token this last one was a comment } case '*': m_inMultipleLineComment = true; if (RawTokens) { // if we are looking for raw tokens, we don't care about important comments // or conditional comments or what-have-you. Scan the comment and return it // as the current token SkipMultilineComment(false); token = JSToken.Comment; break; } else { bool importantComment = false; if (GetChar(++m_currentPos) == '@' && !IgnoreConditionalCompilation && !m_peekModeOn) { // we have /*@ // if we have not turned on conditional-compilation yet, then let's peek to see if the next // few characters are cc_on -- if so, turn it on. if (!m_preProcessorOn) { // we are currently on the @ -- start peeking from there if (!CheckSubstring(m_currentPos + 1, "cc_on")) { // we aren't turning on conditional comments. We need to ignore this comment // as just another multi-line comment SkipMultilineComment(false); goto nextToken; } } // if the NEXT character is not an identifier character, then we need to skip // the @ character -- otherwise leave it there if (!IsValidIdentifierStart(GetChar(m_currentPos + 1))) { ++m_currentPos; } // if we aren't already in a conditional comment if (!m_inConditionalComment) { // we are in one now m_inConditionalComment = true; token = JSToken.ConditionalCommentStart; break; } // we were already in a conditional comment, so ignore the superfluous // conditional comment start goto nextToken; } if (GetChar(m_currentPos) == '!') { // found an "important" comment that we want to preserve importantComment = true; } SkipMultilineComment(importantComment); goto nextToken; // read another token this last one was a comment } default: // if we're not just returning raw tokens, then we don't need to do this logic. // otherwise if the previous token CAN be before a regular expression.... if (RawTokens && RegExpCanFollow(m_previousToken)) { // we think this is probably a regular expression. // if it is... if (ScanRegExp() != null) { // also scan the flags (if any) ScanRegExpFlags(); token = JSToken.RegularExpression; } else if (c == '=') { m_currentPos++; token = JSToken.DivideAssign; } } else if (c == '=') { m_currentPos++; token = JSToken.DivideAssign; } break; } break; case '^': token = JSToken.BitwiseXor; if ('=' == GetChar(m_currentPos)) { m_currentPos++; token = JSToken.BitwiseXorAssign; } break; case '%': token = JSToken.Modulo; if ('=' == GetChar(m_currentPos)) { m_currentPos++; token = JSToken.ModuloAssign; } break; case '(': token = JSToken.LeftParenthesis; break; case ')': token = JSToken.RightParenthesis; break; case '{': token = JSToken.LeftCurly; break; case '}': token = JSToken.RightCurly; break; case '[': token = JSToken.LeftBracket; break; case ']': token = JSToken.RightBracket; break; case ';': token = JSToken.Semicolon; break; case '"': goto case '\''; case '\'': token = JSToken.StringLiteral; ScanString(c); break; // line terminator crap case '\r': // if we are in a single-line conditional comment, then we want // to return the end of comment token WITHOUT moving past the end of line // characters if (m_inConditionalComment && m_inSingleLineComment) { token = JSToken.ConditionalCommentEnd; m_inConditionalComment = m_inSingleLineComment = false; break; } // \r\n is a valid SINGLE line-terminator. So if the \r is // followed by a \n, we only want to process a single line terminator. if (GetChar(m_currentPos) == '\n') { m_currentPos++; } // drop down into normal line-ending processing goto case '\n'; case '\n': case (char)0x2028: case (char)0x2029: // if we are in a single-line conditional comment, then // clean up the flags and return the end of the conditional comment // WITHOUT skipping past the end of line if (m_inConditionalComment && m_inSingleLineComment) { token = JSToken.ConditionalCommentEnd; m_inConditionalComment = m_inSingleLineComment = false; break; } CurrentLine++; StartLinePosition = m_currentPos; m_inSingleLineComment = false; if (RawTokens) { // if we are looking for raw tokens, then return this as the current token token = JSToken.EndOfLine; break; } else { // otherwise eat it and move on goto nextToken; } case '@': if (IgnoreConditionalCompilation) { // if the switch to ignore conditional compilation is on, then we don't know // anything about conditional-compilation statements, and the @-sign character // is illegal at this spot. HandleError(JSError.IllegalChar); if (RawTokens) { // if we are just looking for raw tokens, return this one as an error token token = JSToken.Error; break; } else { // otherwise eat it goto nextToken; } } // we do care about conditional compilation if we get here if (m_peekModeOn) { // but if we're in peek mode, we just need to know WHAT the // next token is, and we don't need to go any deeper. m_currentToken.Token = JSToken.PreprocessDirective; break; } // see if the @-sign is immediately followed by an identifier. If it is, // we'll see which one so we can tell if it's a conditional-compilation statement int startPosition = m_currentPos; m_currentToken.StartPosition = startPosition; m_currentToken.StartLineNumber = CurrentLine; m_currentToken.StartLinePosition = StartLinePosition; ScanIdentifier(); switch (m_currentPos - startPosition) { case 0: // look for '@*/'. if (/*m_preProcessorOn &&*/ '*' == GetChar(m_currentPos) && '/' == GetChar(++m_currentPos)) { m_currentPos++; m_inMultipleLineComment = false; m_inConditionalComment = false; token = JSToken.ConditionalCommentEnd; break; } // otherwise we just have a @ sitting by itself! // throw an error and loop back to the next token. HandleError(JSError.IllegalChar); if (RawTokens) { // if we are just looking for raw tokens, return this one as an error token token = JSToken.Error; break; } else { // otherwise eat it goto nextToken; } case 2: if (CheckSubstring(startPosition, "if")) { token = JSToken.ConditionalCompilationIf; // increment the if-level ++m_ccIfLevel; // if we're not in a conditional comment and we haven't explicitly // turned on conditional compilation when we encounter // a @if statement, then we can implicitly turn it on. if (!m_inConditionalComment && !m_preProcessorOn) { m_preProcessorOn = true; } break; } // the string isn't a known preprocessor command, so // fall into the default processing to handle it as a variable name goto default; case 3: if (CheckSubstring(startPosition, "set")) { token = JSToken.ConditionalCompilationSet; // if we're not in a conditional comment and we haven't explicitly // turned on conditional compilation when we encounter // a @set statement, then we can implicitly turn it on. if (!m_inConditionalComment && !m_preProcessorOn) { m_preProcessorOn = true; } break; } if (CheckSubstring(startPosition, "end")) { token = JSToken.ConditionalCompilationEnd; if (m_ccIfLevel > 0) { // down one more @if level m_ccIfLevel--; } else { // not corresponding @if -- invalid @end statement HandleError(JSError.CCInvalidEnd); } break; } // the string isn't a known preprocessor command, so // fall into the default processing to handle it as a variable name goto default; case 4: if (CheckSubstring(startPosition, "else")) { token = JSToken.ConditionalCompilationElse; // if we don't have a corresponding @if statement, then throw and error // (but keep processing) if (m_ccIfLevel <= 0) { HandleError(JSError.CCInvalidElse); } break; } if (CheckSubstring(startPosition, "elif")) { token = JSToken.ConditionalCompilationElseIf; // if we don't have a corresponding @if statement, then throw and error // (but keep processing) if (m_ccIfLevel <= 0) { HandleError(JSError.CCInvalidElseIf); } break; } // the string isn't a known preprocessor command, so // fall into the default processing to handle it as a variable name goto default; case 5: if (CheckSubstring(startPosition, "cc_on")) { // if we have already turned on conditional compilation.... if (!RawTokens && m_preProcessorOn && EatUnnecessaryCCOn) { // we'll just eat the token here because we don't even // need to expose it to the parser at this time. goto nextToken; } // turn it on and return the @cc_on token m_preProcessorOn = true; token = JSToken.ConditionalCompilationOn; break; } // the string isn't a known preprocessor command, so // fall into the default processing to handle it as a variable name goto default; default: // we have @[id], where [id] is a valid identifier. // if we haven't explicitly turned on conditional compilation, // we'll keep processing, but we need to fire an error to indicate // that the code should turn it on first. if (!m_preProcessorOn) { HandleError(JSError.CCOff); } token = JSToken.PreprocessorConstant; break; } break; case '$': goto case '_'; case '_': ScanIdentifier(); token = JSToken.Identifier; break; default: if ('a' <= c && c <= 'z') { JSKeyword keyword = m_keywords[c - 'a']; if (null != keyword) { token = ScanKeyword(keyword); } else { token = JSToken.Identifier; ScanIdentifier(); } } else if (IsDigit(c)) { token = ScanNumber(c); } else if (IsValidIdentifierStart(c)) { token = JSToken.Identifier; ScanIdentifier(); } else if (RawTokens && IsBlankSpace(c)) { // we are asking for raw tokens, and this is the start of a stretch of whitespace. // advance to the end of the whitespace, and return that as the token while (JSScanner.IsBlankSpace(GetChar(m_currentPos))) { ++m_currentPos; } token = JSToken.WhiteSpace; } else { m_currentToken.EndLineNumber = CurrentLine; m_currentToken.EndLinePosition = StartLinePosition; m_currentToken.EndPosition = m_currentPos; HandleError(JSError.IllegalChar); if (RawTokens) { // if we are just looking for raw tokens, return this one as an error token token = JSToken.Error; break; } else { // otherwise eat it goto nextToken; } } break; } m_currentToken.EndLineNumber = CurrentLine; m_currentToken.EndLinePosition = StartLinePosition; m_currentToken.EndPosition = m_currentPos; GotEndOfLine = (CurrentLine > thisCurrentLine || token == JSToken.EndOfFile) ? true : false; if (GotEndOfLine && token == JSToken.StringLiteral && m_currentToken.StartLineNumber == thisCurrentLine) { GotEndOfLine = false; } } catch (IndexOutOfRangeException) { m_currentToken.Token = JSToken.None; m_currentToken.EndPosition = m_currentPos; m_currentToken.EndLineNumber = CurrentLine; m_currentToken.EndLinePosition = StartLinePosition; throw new ScannerException(JSError.ErrorEndOfFile); } // this is now the current token m_currentToken.Token = token; // if this the kind of token we want to know about the next time, then save it switch(token) { case JSToken.WhiteSpace: case JSToken.AspNetBlock: case JSToken.Comment: case JSToken.UnterminatedComment: case JSToken.ConditionalCompilationOn: case JSToken.ConditionalCompilationSet: case JSToken.ConditionalCompilationIf: case JSToken.ConditionalCompilationElseIf: case JSToken.ConditionalCompilationElse: case JSToken.ConditionalCompilationEnd: // don't save these tokens for next time break; default: m_previousToken = token; break; } //if (TokenCounts.ContainsKey(token)) //{ // ++TokenCounts[token]; //} //else //{ // TokenCounts.Add(token, 1); //} }
private static bool RegExpCanFollow(JSToken previousToken) { switch(previousToken) { case JSToken.Do: case JSToken.Return: case JSToken.Throw: case JSToken.LeftCurly: case JSToken.Semicolon: case JSToken.LeftParenthesis: case JSToken.LeftBracket: case JSToken.ConditionalIf: case JSToken.Colon: case JSToken.Comma: case JSToken.Case: case JSToken.Else: case JSToken.EndOfLine: case JSToken.RightCurly: case JSToken.LogicalNot: case JSToken.BitwiseNot: case JSToken.Delete: case JSToken.Void: case JSToken.New: case JSToken.TypeOf: case JSToken.Increment: case JSToken.Decrement: case JSToken.Plus: case JSToken.Minus: case JSToken.LogicalOr: case JSToken.LogicalAnd: case JSToken.BitwiseOr: case JSToken.BitwiseXor: case JSToken.BitwiseAnd: case JSToken.Equal: case JSToken.NotEqual: case JSToken.StrictEqual: case JSToken.StrictNotEqual: case JSToken.GreaterThan: case JSToken.LessThan: case JSToken.LessThanEqual: case JSToken.GreaterThanEqual: case JSToken.LeftShift: case JSToken.RightShift: case JSToken.UnsignedRightShift: case JSToken.Multiply: case JSToken.Divide: case JSToken.Modulo: case JSToken.InstanceOf: case JSToken.In: case JSToken.Assign: case JSToken.PlusAssign: case JSToken.MinusAssign: case JSToken.MultiplyAssign: case JSToken.DivideAssign: case JSToken.BitwiseAndAssign: case JSToken.BitwiseOrAssign: case JSToken.BitwiseXorAssign: case JSToken.ModuloAssign: case JSToken.LeftShiftAssign: case JSToken.RightShiftAssign: case JSToken.UnsignedRightShiftAssign: case JSToken.None: return true; default: return false; } }
internal static bool IsRightAssociativeOperator(JSToken token) { return JSToken.Assign <= token && token <= JSToken.ConditionalIf; }
// This function return whether an operator is processable in ParseExpression. // Comma is out of this list and so are the unary ops internal static bool IsProcessableOperator(JSToken token) { return JSToken.FirstBinaryOperator <= token && token <= JSToken.ConditionalIf; }
internal static bool IsAssignmentOperator(JSToken token) { return JSToken.Assign <= token && token <= JSToken.LastAssign; }