public Token collectToken() { Loc loc; Token token; string value; Token entry; loc = new Loc() { start = new Loc.Position() { line = lineNumber, column = index - lineStart } }; token = advance(); loc.end = new Loc.Position() { line = lineNumber, column = index - lineStart }; if (token.type != TokenType.EOF) { value = source.Substring(token.start, token.end - token.start); entry = new Token { type = token.type, value = value, range = new Token.TokenRange() { start = token.start, end = token.end }, loc = loc, lineNumber = lineNumber, start = loc.start.column, end = loc.end.column, }; if (token.regex != null) { entry.regex = new Regex() { pattern = token.regex.pattern, flags = token.regex.flags }; } extra.tokens.Add(entry); } return token; }
private dynamic ParseVariableDeclaration(string kind) { var id = ParseVariableIdentifier(); dynamic init = null; var firstToken = _extra.Tokens[_extra.Tokens.Count - 1]; // 12.2.1 if (_strict && IsRestrictedWord(id.Name)) { ThrowErrorTolerant(null, Messages.StrictVarName); } if (kind == "const") { Expect('='); init = ParseAssignmentExpression(); } else if (Match('=')) { Lex(); init = ParseAssignmentExpression(); } Token lastToken = null; if (init != null) { if (init.Type != "ArrayExpression") { lastToken = new Token { Range = new Token.TokenRange { Start = init.Range.Start, End = init.Range.End }, Loc = new Token.TokenLoc { Start = new Token.TokenLoc.TokenPosition { Line = init.Loc.Start.Line, Column = init.Loc.Start.Column }, End = new Token.TokenLoc.TokenPosition { Line = init.Loc.End.Line, Column = init.Loc.End.Column } } }; } } else lastToken = firstToken; if (lastToken != null) { return new VariableDeclarator(_codeGeneration) { Id = id, Init = init, Range = new Range {Start = firstToken.Range.Start, End = lastToken.Range.End}, Loc = new Loc { Start = new Loc.Position { Line = firstToken.Loc.Start.Line, Column = firstToken.Loc.Start.Column }, End = new Loc.Position {Line = lastToken.Loc.End.Line, Column = lastToken.Loc.End.Column} } }; } else { return new VariableDeclarator(_codeGeneration) { Id = id, Init = init }; } }
private object CreateRawLiteral(Token token) { //return new Token { // Type= Syntax.Literal, // Value: token.value, // Raw: sliceSource(token.range[0], token.range[1]) //}; return null; }
private void ThrowError(Token token, string msg) { Error error = null; //if (typeof token.lineNumber === 'number') { if (token != null) { error = new Error("Line " + token.LineNumber + ": " + msg) { Index = token.Range.Start, LineNumber = token.LineNumber, Column = token.Range.Start - _lineStart + 1 }; } else { error = new Error("Line " + _lineNumber + ": " + msg) { Index = _index, LineNumber = _lineNumber, Column = _index - _lineStart + 1 }; } throw error; }
// Throw an exception because of the token. private void ThrowUnexpected(Token token) { if (token.Type == TokenType.EOF) { ThrowError(token, Messages.UnexpectedEOS); } if (token.Type == TokenType.NumericLiteral) { ThrowError(token, Messages.UnexpectedNumber); } if (token.Type == TokenType.StringLiteral) { ThrowError(token, Messages.UnexpectedString); } if (token.Type == TokenType.Identifier) { ThrowError(token, Messages.UnexpectedIdentifier); } if (token.Type == TokenType.Keyword) { if (IsFutureReservedWord(token.Value)) { ThrowError(token, Messages.UnexpectedReserved); } else if (_strict && IsStrictModeReservedWord(token.Value)) { ThrowError(token, Messages.StrictReservedWord); } ThrowError(token, string.Format(Messages.UnexpectedToken, token.Value)); } // BooleanLiteral, NullLiteral, or Punctuator. ThrowError(token, string.Format(Messages.UnexpectedToken, token.Value)); }
private Token ScanRegExp() { var str = ""; //, ch, start, pattern, flags, value, classMarker = false, restore, terminated = false; var classMarker = false; var terminated = false; string pattern; int restore; var value = ""; _buffer = null; SkipComment(); var start = _index; var ch = _source[_index]; Assert(ch == '/', "Regular expression literal must start with a slash"); str = NextChar().ToString(); while (_index < _length) { ch = NextChar(); str += ch; if (classMarker) { if (ch == ']') { classMarker = false; } } else { if (ch == '\\') { ch = NextChar(); // ECMA-262 7.8.5 if (IsLineTerminator(ch)) { ThrowError(null, Messages.UnterminatedRegExp); } str += ch; } else if (ch == '/') { terminated = true; break; } else if (ch == '[') { classMarker = true; } else if (IsLineTerminator(ch)) { ThrowError(null, Messages.UnterminatedRegExp); } } } if (!terminated) { ThrowError(null, Messages.UnterminatedRegExp); } // Exclude leading and trailing slash. pattern = str.Substring(1, str.Length - 2); var flags = '\0'; while (_index < _length) { ch = _source[_index]; if (!IsIdentifierPart(ch)) { break; } ++_index; if (ch == '\\' && _index < _length) { ch = _source[_index]; if (ch == 'u') { ++_index; restore = _index; ch = ScanHexEscape('u'); if (ch != '\0') { flags += ch; str += "\\u"; for (; restore < _index; ++restore) { str += _source[restore]; } } else { _index = restore; flags += 'u'; str += "\\u"; } } else { str += '\\'; } } else { flags += ch; str += ch; } } try { value = new System.Text.RegularExpressions.Regex(pattern).Match(flags.ToString()).Value; } catch (Exception e) { ThrowError(null, Messages.InvalidRegExp); } return new Token { Literal = str, Value = value, Range = new Token.TokenRange() { Start = start, End = _index } }; }
private Token Lex() { if (_buffer != null) { _index = _buffer.Range.End; _lineNumber = _buffer.LineNumber; _lineStart = _buffer.LineStart; var token = _buffer; _buffer = null; return token; } _buffer = null; return Advance(); }
public void tolerateUnexpectedToken(Token token = null, string message = "") { var error = unexpectedTokenError(token, message); if (extra.errors.Any()) { recordError(error); } else { throw error; } }
// Cover grammar support. // // When an assignment expression position starts with an left parenthesis, the determination of the type // of the syntax is to be deferred arbitrarily long until the end of the parentheses pair (plus a lookahead) // or the first comma. This situation also defers the determination of all the expressions nested in the pair. // // There are three productions that can be parsed in a parentheses pair that needs to be determined // after the outermost pair is closed. They are: // // 1. AssignmentExpression // 2. BindingElements // 3. AssignmentTargets // // In order to avoid exponential backtracking, we use two flags to denote if the production can be // binding element or assignment target. // // The three productions have the relationship: // // BindingElements ? AssignmentTargets ? AssignmentExpression // // with a single exception that CoverInitializedName when used directly in an Expression, generates // an early error. Therefore, we need the third state, firstCoverInitializedNameError, to track the // first usage of CoverInitializedName and report it when we reached the end of the parentheses pair. // // isolateCoverGrammar function runs the given parser function with a new cover grammar context, and it does not // effect the current flags. This means the production the parser parses is only used as an expression. Therefore // the CoverInitializedName check is conducted. // // inheritCoverGrammar function runs the given parse function with a new cover grammar context, and it propagates // the flags outside of the parser. This means the production the parser parses is used as a part of a potential // pattern. The CoverInitializedName check is deferred. public Node isolateCoverGrammar(Func<Node> parser) { bool oldIsBindingElement = isBindingElement; bool oldIsAssignmentTarget = isAssignmentTarget; Token oldFirstCoverInitializedNameError = firstCoverInitializedNameError; Node result; isBindingElement = true; isAssignmentTarget = true; firstCoverInitializedNameError = null; result = parser(); if (firstCoverInitializedNameError != null) { throwUnexpectedToken(firstCoverInitializedNameError); } isBindingElement = oldIsBindingElement; isAssignmentTarget = oldIsAssignmentTarget; firstCoverInitializedNameError = oldFirstCoverInitializedNameError; return result; }
// Throw an exception because of the token. public Error unexpectedTokenError(Token token, string message = null) { string value; string msg = message ?? Messages.UnexpectedToken; if (token != null) { if (message != null) { msg = (token.type == TokenType.EOF) ? Messages.UnexpectedEOS : (token.type == TokenType.Identifier) ? Messages.UnexpectedIdentifier : (token.type == TokenType.NumericLiteral) ? Messages.UnexpectedNumber : (token.type == TokenType.StringLiteral) ? Messages.UnexpectedString : (token.type == TokenType.Template) ? Messages.UnexpectedTemplate : Messages.UnexpectedToken; if (token.type == TokenType.Keyword) { if (isFutureReservedWord(token.value)) { msg = Messages.UnexpectedReserved; } else if (strict && isStrictModeReservedWord(token.value)) { msg = Messages.StrictReservedWord; } } } // value = (token.type == TokenType.Template) ? token.value.raw : token.value; value = token.value; } else { value = "ILLEGAL"; } //msg = msg.replace("%0", value); return createError(token.lineNumber, token.start, msg); //: // createError(scanning ? lineNumber : lastLineNumber, scanning ? index : lastIndex, msg); }
public void throwUnexpectedToken(Token token = null, string message = "") { throw unexpectedTokenError(token, message); }
public static Loc WrappingSourceLocation(Token startToken) { return new Loc() { start = new Loc.Position() { line = startToken.lineNumber, column = startToken.start - startToken.lineStart }, end = null }; }
public void peek() { scanning = true; skipComment(); lastIndex = index; lastLineNumber = lineNumber; lastLineStart = lineStart; startIndex = index; startLineNumber = lineNumber; startLineStart = lineStart; lookahead = extra.tokens.Count >= 0 ? collectToken() : advance(); scanning = false; }
public Token lex() { Token token; scanning = true; lastIndex = index; lastLineNumber = lineNumber; lastLineStart = lineStart; skipComment(); token = lookahead; startIndex = index; startLineNumber = lineNumber; startLineStart = lineStart; lookahead = extra.tokens.Count >= 0 ? collectToken() : advance(); scanning = false; return token; }
public Node parse(string code, Options options) { // Program program, toString; //toString = String; //if (typeof code !== "string" && !(code instanceof String)) { // code = toString(code); //} source = code; index = 0; lineNumber = (source.Length > 0) ? 1 : 0; lineStart = 0; startIndex = index; startLineNumber = lineNumber; startLineStart = lineStart; length = source.Length; lookahead = null; state = new State() { allowIn = true, allowYield = true, labelSet = new List<string>(), inFunctionBody = false, inIteration = false, inSwitch = false, lastCommentStart = -1, curlyStack = new Stack<string>(), sourceType = "script" }; strict = false; extra = new Extra(); if (options != null) { extra.range = options.range; extra.loc = options.loc; extra.attachComment = options.attachComment; //if (extra.loc && options.source != null && options.source != undefined) { // extra.source = toString(options.source); //} if (options.tokens) { extra.tokens = new List<Token>(); } if (options.comment) { extra.comments = new List<Comment>(); } if (options.tolerant) { extra.errors = new List<Error>(); } if (extra.attachComment) { extra.range = true; extra.comments = new List<Comment>(); extra.bottomRightStack = new List<Token>(); extra.trailingComments = new List<Comment>(); extra.leadingComments = new List<Comment>(); } if (options.sourceType == "module") { // very restrictive condition for now state.sourceType = options.sourceType; strict = true; } } //try //{ var program = parseProgram(); //if (typeof extra.comments !== "undefined") { // program.comments = extra.comments; //} //if (typeof extra.tokens !== "undefined") { // filterTokenLocation(); // program.tokens = extra.tokens; //} //if (typeof extra.errors !== "undefined") { // program.errors = extra.errors; //} //} //catch (Exception e) //{ // throw e; //} //finally //{ // extra = new Extra(); //} return program; }
public Node inheritCoverGrammar(Func<Node> parser) { bool oldIsBindingElement = isBindingElement; bool oldIsAssignmentTarget = isAssignmentTarget; Token oldFirstCoverInitializedNameError = firstCoverInitializedNameError; Node result; isBindingElement = true; isAssignmentTarget = true; firstCoverInitializedNameError = null; result = parser(); isBindingElement = isBindingElement && oldIsBindingElement; isAssignmentTarget = isAssignmentTarget && oldIsAssignmentTarget; firstCoverInitializedNameError = oldFirstCoverInitializedNameError ?? firstCoverInitializedNameError; return result; }
// ECMA-262 11.7 Punctuators public Token scanPunctuator() { Token token; string str; token = new Token() { type = TokenType.Punctuator, value = "", lineNumber = lineNumber, lineStart = lineStart, start = index, end = index }; // Check for most common single-character punctuators. str = source.Substring(index, 1); switch (str) { case "(": if (extra.tokenize) { extra.openParenToken = extra.tokens.Count; } ++index; break; case "{": if (extra.tokenize) { extra.openCurlyToken = extra.tokens.Count; } state.curlyStack.Push("{"); ++index; break; case ".": ++index; if (source[index] == '.' && source[index + 1] == '.') { // Spread operator: ... index += 2; str = "..."; } break; case "}": ++index; state.curlyStack.Pop(); break; case ")": case ";": case ",": case "[": case "]": case ":": case "?": case "~": ++index; break; default: // 4-character punctuator. str = source.Substring(index, 4); if (str == ">>>=") { index += 4; } else { // 3-character punctuators. str = str.Substring(0, 3); if (str == "==" || str == "!==" || str == ">>>" || str == "<<=" || str == ">>=") { index += 3; } else { // 2-character punctuators. str = str.Substring(0, 2); if (str == "&&" || str == "||" || str == "==" || str == "!=" || str == "+=" || str == "-=" || str == "*=" || str == "/=" || str == "++" || str == "--" || str == "<<" || str == ">>" || str == "&=" || str == "|=" || str == "^=" || str == "%=" || str == "<=" || str == ">=" || str == "=>") { index += 2; } else { // 1-character punctuators. str = source[index].ToString(); if ("<>=!+-*%&|^/".IndexOf(str) >= 0) { ++index; } } } } break; } if (index == token.start) { throwUnexpectedToken(); } token.end = index; token.value = str; return token; }
// This function is to try to parse a MethodDefinition as defined in 14.3. But in the case of object literals, // it might be called at a position where there is in fact a short hand identifier pattern or a data property. // This can only be determined after we consumed up to the left parentheses. // // In order to avoid back tracking, it returns `null` if the position is not a MethodDefinition and the caller // is responsible to visit other options. public Node tryParseMethodDefinition(Token token, Node key, bool computed, Node node) { Node value; Options options; Node methodNode; Options @params; bool previousAllowYield = state.allowYield; if (token.type == TokenType.Identifier) { // check for `get` and `set`; if (token.value == "get" && lookaheadPropertyName()) { computed = match("["); key = parseObjectPropertyKey(); methodNode = new Node(); expect("("); expect(")"); state.allowYield = false; value = parsePropertyFunction(methodNode, new Options() { @params = new List<Node>(), defaults = new List<Node>(), stricted = null, firstRestricted = null, message = null }, false); state.allowYield = previousAllowYield; return node.finishProperty("get", key, computed, value, false, false); } else if (token.value == "set" && lookaheadPropertyName()) { computed = match("["); key = parseObjectPropertyKey(); methodNode = new Node(); expect("("); options = new Options() { @params = new List<Node>(), defaultCount = 0, defaults = new List<Node>(), firstRestricted = null, // paramSet = {} }; if (match(")")) { tolerateUnexpectedToken(lookahead); } else { state.allowYield = false; parseParam(options); state.allowYield = previousAllowYield; if (options.defaultCount == 0) { options.defaults = new List<Node>(); } } expect(")"); state.allowYield = false; value = parsePropertyFunction(methodNode, options, false); state.allowYield = previousAllowYield; return node.finishProperty("set", key, computed, value, false, false); } } else if (token.type == TokenType.Punctuator && token.value == "*" && lookaheadPropertyName()) { computed = match("["); key = parseObjectPropertyKey(); methodNode = new Node(); state.allowYield = true; @params = parseParams(); state.allowYield = previousAllowYield; state.allowYield = false; value = parsePropertyFunction(methodNode, @params, true); state.allowYield = previousAllowYield; return node.finishProperty("init", key, computed, value, true, false); } if (key != null && match("(")) { value = parsePropertyMethodFunction(); return node.finishProperty("init", key, computed, value, true, false); } // Not a MethodDefinition. return null; }
private bool IsIdentifierName(Token token) { return token.Type == TokenType.Identifier || token.Type == TokenType.Keyword || token.Type == TokenType.BooleanLiteral || token.Type == TokenType.NullLiteral; }
public Node parseObjectProperty( bool hasProto) { var token = lookahead; Node node = new Node(); bool computed; Node key = null; Node maybeMethod; bool proto; Node value; computed = match("["); if (match("*")) { lex(); } else { key = parseObjectPropertyKey(); } maybeMethod = tryParseMethodDefinition(token, key, computed, node); if (maybeMethod!= null) { return maybeMethod; } if (key== null) { throwUnexpectedToken(lookahead); } // Check for duplicated __proto__ if (!computed) { proto = (key.type == Syntax.Identifier && key.name == "__proto__") || (key.type == Syntax.Literal && key.name== "__proto__"); if (hasProto && proto) { tolerateError(Messages.DuplicateProtoProperty); } hasProto |= proto; } if (match(":")) { lex(); value = inheritCoverGrammar(parseAssignmentExpression); return node.finishProperty("init", key, computed, value, false, false); } if (token.type == TokenType.Identifier) { if (match("=")) { firstCoverInitializedNameError = lookahead; lex(); value = isolateCoverGrammar(parseAssignmentExpression); return node.finishProperty("init", key, computed, new Node(token).finishAssignmentPattern(key, value), false, true); } return node.finishProperty("init", key, computed, key, false, true); } throwUnexpectedToken(lookahead); return null; }
private Token Lookahead() { if (_buffer != null) { return _buffer; } var pos = _index; var line = _lineNumber; var start = _lineStart; _buffer = Advance(); _index = pos; _lineNumber = line; _lineStart = start; return _buffer; }
public int binaryPrecedence(Token token, bool allowIn) { var prec = 0; if (token.type != TokenType.Punctuator && token.type != TokenType.Keyword) { return 0; } switch (token.value.ToString()) { case "||": prec = 1; break; case "&&": prec = 2; break; case "|": prec = 3; break; case "^": prec = 4; break; case "&": prec = 5; break; case "==": case "!=": //case "==": case "!==": prec = 6; break; case "<": case ">": case "<=": case ">=": case "instanceof": prec = 7; break; case "in": prec = allowIn ? 7 : 0; break; case "<<": case ">>": case ">>>": prec = 8; break; case "+": case "-": prec = 9; break; case "*": case "/": case "%": prec = 11; break; default: break; } return prec; }
private void ThrowErrorTolerant(Token token, string msg) { try { ThrowError(token, msg); } catch (Exception e) { if (_extra.Errors.Count > 0) { _extra.Errors.Add(e); } else { throw e; } } }
// ECMA-262 12.14 Assignment Operators public Node parseAssignmentExpression() { Token token; Node expr; Node right; Options list; Token startToken; startToken = lookahead; token = lookahead; if (!state.allowYield && matchKeyword("yield")) { return parseYieldExpression(); } expr = parseConditionalExpression(); if (expr.type == PlaceHolders.ArrowParameterPlaceHolder || match("=>")) { isAssignmentTarget = isBindingElement = false; list = reinterpretAsCoverFormalsList(expr); if (list != null) { firstCoverInitializedNameError = null; return parseArrowFunctionExpression(list, new Node(startToken)); } return expr; } if (matchAssign()) { if (!isAssignmentTarget) { tolerateError(Messages.InvalidLHSInAssignment); } // ECMA-262 12.1.1 if (strict && expr.type == Syntax.Identifier) { if (isRestrictedWord(expr.name)) { tolerateUnexpectedToken(token, Messages.StrictLHSAssignment); } if (isStrictModeReservedWord(expr.name)) { tolerateUnexpectedToken(token, Messages.StrictReservedWord); } } if (!match("=")) { isAssignmentTarget = isBindingElement = false; } else { reinterpretExpressionAsPattern(expr); } token = lex(); right = isolateCoverGrammar(parseAssignmentExpression); expr = new Node(startToken).finishAssignmentExpression(token.value, expr, right); firstCoverInitializedNameError = null; } return expr; }
// 11.1.5 Object Initialiser private dynamic ParsePropertyFunction(List<Parameter> param, Token first) { //var previousStrict, body; var previousStrict = _strict; var body = ParseFunctionSourceElements(); if (first != null && _strict && IsRestrictedWord(param[0].Name)) { ThrowError(first, Messages.StrictParamName); } _strict = previousStrict; return new { Type = Syntax.FunctionExpression, //Id = null, Params = param, //Defaults = null, Body = body, //Rest = null, Generator = false, IsExpression = false }; }
public Options parseParams(Token firstRestricted = null) { Options options; options = new Options { @params = new List<Node>(), defaultCount = 0, defaults = new List<Node>(), firstRestricted = firstRestricted }; expect("("); if (!match(")")) { //options.paramSet = //{ //} //; while (startIndex < length) { if (!parseParam(options)) { break; } expect(","); } } expect(")"); if (options.defaultCount == 0) { options.defaults = new List<Node>(); } return new Options() { @params = options.@params, defaults = options.defaults, stricted = options.stricted, firstRestricted = options.firstRestricted, message = options.message }; }
private dynamic CreateLiteral(Token token) { return new Literal(_codeGeneration) { IsString = token.Type == TokenType.StringLiteral, Value = token.Value, Range = new Range { Start = token.Range.Start, End = token.Range.End }, Loc = new Loc { Start = new Loc.Position { Line = token.Loc.Start.Line, Column = token.Loc.Start.Column }, End = new Loc.Position { Line = token.Loc.End.Line, Column = token.Loc.End.Column } } }; }
public List<Token> tokenize(string code, Options options) { //var toString; List<Token> tokens; //toString = String; //if (typeof code !== "string" && !(code instanceof String)) { // code = toString(code); //} source = code; index = 0; lineNumber = (source.Length > 0) ? 1 : 0; lineStart = 0; startIndex = index; startLineNumber = lineNumber; startLineStart = lineStart; length = source.Length; lookahead = null; state = new State() { allowIn = true, allowYield = true, labelSet = new List<string>(), inFunctionBody = false, inIteration = false, inSwitch = false, lastCommentStart = -1, curlyStack = new Stack<string>() }; extra = new Extra(); // Options matching. options = options ?? new Options(); // Of course we collect tokens here. options.tokens = true; extra.tokens = new List<Token>(); extra.tokenize = true; // The following two fields are necessary to compute the Regex tokens. extra.openParenToken = -1; extra.openCurlyToken = -1; extra.range = options.range; extra.loc = options.loc; if (options.comment) { extra.comments = new List<Comment>(); } if (options.tolerant) { extra.errors = new List<Error>(); } //try //{ peek(); if (lookahead.type == TokenType.EOF) { return extra.tokens; } lex(); while (lookahead.type != TokenType.EOF) { try { lex(); } catch (Error lexError) { if (extra.errors != null) { recordError(lexError); // We have to break on the first error // to avoid infinite loops. break; } else { throw lexError; } } } filterTokenLocation(); tokens = extra.tokens; //tokens.comments = extra.comments; //tokens.errors = extra.errors; //} //catch (Exception e) //{ // throw e; //} //finally //{ // extra = new Extra(); //} return tokens; }
public Program Parse(ICodeGeneration codeGeneration, string code) { _codeGeneration = codeGeneration; _lineNumber = (code.Length > 0) ? 1 : 0; _lineStart = 0; _length = code.Length; _buffer = null; _state = new State { AllowIn = true, LabelSet = new Dictionary<string, object>(), LastParenthesized = null, InFunctionBody = false, InIteration = false, InSwitch = false }; _extra = new Extra(); if (_length > 0) { _source = StringToArray(code).ToList(); } return ParseProgram(); }
public bool isIdentifierName(Token token) { return token.type == TokenType.Identifier || token.type == TokenType.Keyword || token.type == TokenType.BooleanLiteral || token.type == TokenType.NullLiteral; }