public Token collectToken()
{
Loc loc;
Token token;
string value;
Token entry;
loc = new Loc()
{
start = new Loc.Position()
{
line = lineNumber,
column = index - lineStart
}
};
token = advance();
loc.end = new Loc.Position()
{
line = lineNumber,
column = index - lineStart
};
if (token.type != TokenType.EOF)
{
value = source.Substring(token.start, token.end - token.start);
entry = new Token
{
type = token.type,
value = value,
range = new Token.TokenRange() { start = token.start, end = token.end },
loc = loc,
lineNumber = lineNumber,
start = loc.start.column,
end = loc.end.column,
};
if (token.regex != null)
{
entry.regex = new Regex()
{
pattern = token.regex.pattern,
flags = token.regex.flags
};
}
extra.tokens.Add(entry);
}
return token;
}
private dynamic ParseVariableDeclaration(string kind)
{
var id = ParseVariableIdentifier();
dynamic init = null;
var firstToken = _extra.Tokens[_extra.Tokens.Count - 1];
// 12.2.1
if (_strict && IsRestrictedWord(id.Name))
{
ThrowErrorTolerant(null, Messages.StrictVarName);
}
if (kind == "const")
{
Expect('=');
init = ParseAssignmentExpression();
}
else if (Match('='))
{
Lex();
init = ParseAssignmentExpression();
}
Token lastToken = null;
if (init != null)
{
if (init.Type != "ArrayExpression")
{
lastToken = new Token
{
Range = new Token.TokenRange
{
Start = init.Range.Start,
End = init.Range.End
},
Loc = new Token.TokenLoc
{
Start = new Token.TokenLoc.TokenPosition
{
Line = init.Loc.Start.Line,
Column = init.Loc.Start.Column
},
End = new Token.TokenLoc.TokenPosition
{
Line = init.Loc.End.Line,
Column = init.Loc.End.Column
}
}
};
}
}
else
lastToken = firstToken;
if (lastToken != null)
{
return new VariableDeclarator(_codeGeneration)
{
Id = id,
Init = init,
Range = new Range {Start = firstToken.Range.Start, End = lastToken.Range.End},
Loc =
new Loc
{
Start =
new Loc.Position
{
Line = firstToken.Loc.Start.Line,
Column = firstToken.Loc.Start.Column
},
End =
new Loc.Position
{Line = lastToken.Loc.End.Line, Column = lastToken.Loc.End.Column}
}
};
}
else
{
return new VariableDeclarator(_codeGeneration)
{
Id = id,
Init = init
};
}
}
private object CreateRawLiteral(Token token)
{
//return new Token {
// Type= Syntax.Literal,
// Value: token.value,
// Raw: sliceSource(token.range[0], token.range[1])
//};
return null;
}
private void ThrowError(Token token, string msg)
{
Error error = null;
//if (typeof token.lineNumber === 'number') {
if (token != null)
{
error = new Error("Line " + token.LineNumber + ": " + msg)
{
Index = token.Range.Start,
LineNumber = token.LineNumber,
Column = token.Range.Start - _lineStart + 1
};
}
else
{
error = new Error("Line " + _lineNumber + ": " + msg)
{
Index = _index,
LineNumber = _lineNumber,
Column = _index - _lineStart + 1
};
}
throw error;
}
// Throw an exception because of the token.
private void ThrowUnexpected(Token token)
{
if (token.Type == TokenType.EOF)
{
ThrowError(token, Messages.UnexpectedEOS);
}
if (token.Type == TokenType.NumericLiteral)
{
ThrowError(token, Messages.UnexpectedNumber);
}
if (token.Type == TokenType.StringLiteral)
{
ThrowError(token, Messages.UnexpectedString);
}
if (token.Type == TokenType.Identifier)
{
ThrowError(token, Messages.UnexpectedIdentifier);
}
if (token.Type == TokenType.Keyword)
{
if (IsFutureReservedWord(token.Value))
{
ThrowError(token, Messages.UnexpectedReserved);
}
else if (_strict && IsStrictModeReservedWord(token.Value))
{
ThrowError(token, Messages.StrictReservedWord);
}
ThrowError(token, string.Format(Messages.UnexpectedToken, token.Value));
}
// BooleanLiteral, NullLiteral, or Punctuator.
ThrowError(token, string.Format(Messages.UnexpectedToken, token.Value));
}
private Token ScanRegExp()
{
var str = ""; //, ch, start, pattern, flags, value, classMarker = false, restore, terminated = false;
var classMarker = false;
var terminated = false;
string pattern;
int restore;
var value = "";
_buffer = null;
SkipComment();
var start = _index;
var ch = _source[_index];
Assert(ch == '/', "Regular expression literal must start with a slash");
str = NextChar().ToString();
while (_index < _length)
{
ch = NextChar();
str += ch;
if (classMarker)
{
if (ch == ']')
{
classMarker = false;
}
}
else
{
if (ch == '\\')
{
ch = NextChar();
// ECMA-262 7.8.5
if (IsLineTerminator(ch))
{
ThrowError(null, Messages.UnterminatedRegExp);
}
str += ch;
}
else if (ch == '/')
{
terminated = true;
break;
}
else if (ch == '[')
{
classMarker = true;
}
else if (IsLineTerminator(ch))
{
ThrowError(null, Messages.UnterminatedRegExp);
}
}
}
if (!terminated)
{
ThrowError(null, Messages.UnterminatedRegExp);
}
// Exclude leading and trailing slash.
pattern = str.Substring(1, str.Length - 2);
var flags = '\0';
while (_index < _length)
{
ch = _source[_index];
if (!IsIdentifierPart(ch))
{
break;
}
++_index;
if (ch == '\\' && _index < _length)
{
ch = _source[_index];
if (ch == 'u')
{
++_index;
restore = _index;
ch = ScanHexEscape('u');
if (ch != '\0')
{
flags += ch;
str += "\\u";
for (; restore < _index; ++restore)
{
str += _source[restore];
}
}
else
{
_index = restore;
flags += 'u';
str += "\\u";
}
}
else
{
str += '\\';
}
}
else
{
flags += ch;
str += ch;
}
}
try
{
value = new System.Text.RegularExpressions.Regex(pattern).Match(flags.ToString()).Value;
}
catch (Exception e)
{
ThrowError(null, Messages.InvalidRegExp);
}
return new Token
{
Literal = str,
Value = value,
Range = new Token.TokenRange() { Start = start, End = _index }
};
}
private Token Lex()
{
if (_buffer != null)
{
_index = _buffer.Range.End;
_lineNumber = _buffer.LineNumber;
_lineStart = _buffer.LineStart;
var token = _buffer;
_buffer = null;
return token;
}
_buffer = null;
return Advance();
}
public void tolerateUnexpectedToken(Token token = null, string message = "")
{
var error = unexpectedTokenError(token, message);
if (extra.errors.Any())
{
recordError(error);
}
else
{
throw error;
}
}
// Cover grammar support.
//
// When an assignment expression position starts with an left parenthesis, the determination of the type
// of the syntax is to be deferred arbitrarily long until the end of the parentheses pair (plus a lookahead)
// or the first comma. This situation also defers the determination of all the expressions nested in the pair.
//
// There are three productions that can be parsed in a parentheses pair that needs to be determined
// after the outermost pair is closed. They are:
//
// 1. AssignmentExpression
// 2. BindingElements
// 3. AssignmentTargets
//
// In order to avoid exponential backtracking, we use two flags to denote if the production can be
// binding element or assignment target.
//
// The three productions have the relationship:
//
// BindingElements ? AssignmentTargets ? AssignmentExpression
//
// with a single exception that CoverInitializedName when used directly in an Expression, generates
// an early error. Therefore, we need the third state, firstCoverInitializedNameError, to track the
// first usage of CoverInitializedName and report it when we reached the end of the parentheses pair.
//
// isolateCoverGrammar function runs the given parser function with a new cover grammar context, and it does not
// effect the current flags. This means the production the parser parses is only used as an expression. Therefore
// the CoverInitializedName check is conducted.
//
// inheritCoverGrammar function runs the given parse function with a new cover grammar context, and it propagates
// the flags outside of the parser. This means the production the parser parses is used as a part of a potential
// pattern. The CoverInitializedName check is deferred.
public Node isolateCoverGrammar(Func<Node> parser)
{
bool oldIsBindingElement = isBindingElement;
bool oldIsAssignmentTarget = isAssignmentTarget;
Token oldFirstCoverInitializedNameError = firstCoverInitializedNameError;
Node result;
isBindingElement = true;
isAssignmentTarget = true;
firstCoverInitializedNameError = null;
result = parser();
if (firstCoverInitializedNameError != null)
{
throwUnexpectedToken(firstCoverInitializedNameError);
}
isBindingElement = oldIsBindingElement;
isAssignmentTarget = oldIsAssignmentTarget;
firstCoverInitializedNameError = oldFirstCoverInitializedNameError;
return result;
}
// Throw an exception because of the token.
public Error unexpectedTokenError(Token token, string message = null)
{
string value;
string msg = message ?? Messages.UnexpectedToken;
if (token != null)
{
if (message != null)
{
msg = (token.type == TokenType.EOF)
? Messages.UnexpectedEOS
: (token.type == TokenType.Identifier)
? Messages.UnexpectedIdentifier
: (token.type == TokenType.NumericLiteral)
? Messages.UnexpectedNumber
: (token.type == TokenType.StringLiteral)
? Messages.UnexpectedString
: (token.type == TokenType.Template)
? Messages.UnexpectedTemplate
: Messages.UnexpectedToken;
if (token.type == TokenType.Keyword)
{
if (isFutureReservedWord(token.value))
{
msg = Messages.UnexpectedReserved;
}
else if (strict && isStrictModeReservedWord(token.value))
{
msg = Messages.StrictReservedWord;
}
}
}
// value = (token.type == TokenType.Template) ? token.value.raw : token.value;
value = token.value;
}
else
{
value = "ILLEGAL";
}
//msg = msg.replace("%0", value);
return createError(token.lineNumber, token.start, msg); //:
// createError(scanning ? lineNumber : lastLineNumber, scanning ? index : lastIndex, msg);
}
public void throwUnexpectedToken(Token token = null, string message = "")
{
throw unexpectedTokenError(token, message);
}
public static Loc WrappingSourceLocation(Token startToken)
{
return new Loc()
{
start = new Loc.Position()
{
line = startToken.lineNumber,
column = startToken.start - startToken.lineStart
},
end = null
};
}
public void peek()
{
scanning = true;
skipComment();
lastIndex = index;
lastLineNumber = lineNumber;
lastLineStart = lineStart;
startIndex = index;
startLineNumber = lineNumber;
startLineStart = lineStart;
lookahead = extra.tokens.Count >= 0 ? collectToken() : advance();
scanning = false;
}
public Token lex()
{
Token token;
scanning = true;
lastIndex = index;
lastLineNumber = lineNumber;
lastLineStart = lineStart;
skipComment();
token = lookahead;
startIndex = index;
startLineNumber = lineNumber;
startLineStart = lineStart;
lookahead = extra.tokens.Count >= 0 ? collectToken() : advance();
scanning = false;
return token;
}
public Node parse(string code, Options options)
{
// Program program, toString;
//toString = String;
//if (typeof code !== "string" && !(code instanceof String)) {
// code = toString(code);
//}
source = code;
index = 0;
lineNumber = (source.Length > 0) ? 1 : 0;
lineStart = 0;
startIndex = index;
startLineNumber = lineNumber;
startLineStart = lineStart;
length = source.Length;
lookahead = null;
state = new State()
{
allowIn = true,
allowYield = true,
labelSet = new List<string>(),
inFunctionBody = false,
inIteration = false,
inSwitch = false,
lastCommentStart = -1,
curlyStack = new Stack<string>(),
sourceType = "script"
};
strict = false;
extra = new Extra();
if (options != null)
{
extra.range = options.range;
extra.loc = options.loc;
extra.attachComment = options.attachComment;
//if (extra.loc && options.source != null && options.source != undefined) {
// extra.source = toString(options.source);
//}
if (options.tokens)
{
extra.tokens = new List<Token>();
}
if (options.comment)
{
extra.comments = new List<Comment>();
}
if (options.tolerant)
{
extra.errors = new List<Error>();
}
if (extra.attachComment)
{
extra.range = true;
extra.comments = new List<Comment>();
extra.bottomRightStack = new List<Token>();
extra.trailingComments = new List<Comment>();
extra.leadingComments = new List<Comment>();
}
if (options.sourceType == "module")
{
// very restrictive condition for now
state.sourceType = options.sourceType;
strict = true;
}
}
//try
//{
var program = parseProgram();
//if (typeof extra.comments !== "undefined") {
// program.comments = extra.comments;
//}
//if (typeof extra.tokens !== "undefined") {
// filterTokenLocation();
// program.tokens = extra.tokens;
//}
//if (typeof extra.errors !== "undefined") {
// program.errors = extra.errors;
//}
//}
//catch (Exception e)
//{
// throw e;
//}
//finally
//{
// extra = new Extra();
//}
return program;
}
public Node inheritCoverGrammar(Func<Node> parser)
{
bool oldIsBindingElement = isBindingElement;
bool oldIsAssignmentTarget = isAssignmentTarget;
Token oldFirstCoverInitializedNameError = firstCoverInitializedNameError;
Node result;
isBindingElement = true;
isAssignmentTarget = true;
firstCoverInitializedNameError = null;
result = parser();
isBindingElement = isBindingElement && oldIsBindingElement;
isAssignmentTarget = isAssignmentTarget && oldIsAssignmentTarget;
firstCoverInitializedNameError = oldFirstCoverInitializedNameError ?? firstCoverInitializedNameError;
return result;
}
// ECMA-262 11.7 Punctuators
public Token scanPunctuator()
{
Token token;
string str;
token = new Token()
{
type = TokenType.Punctuator,
value = "",
lineNumber = lineNumber,
lineStart = lineStart,
start = index,
end = index
};
// Check for most common single-character punctuators.
str = source.Substring(index, 1);
switch (str)
{
case "(":
if (extra.tokenize)
{
extra.openParenToken = extra.tokens.Count;
}
++index;
break;
case "{":
if (extra.tokenize)
{
extra.openCurlyToken = extra.tokens.Count;
}
state.curlyStack.Push("{");
++index;
break;
case ".":
++index;
if (source[index] == '.' && source[index + 1] == '.')
{
// Spread operator: ...
index += 2;
str = "...";
}
break;
case "}":
++index;
state.curlyStack.Pop();
break;
case ")":
case ";":
case ",":
case "[":
case "]":
case ":":
case "?":
case "~":
++index;
break;
default:
// 4-character punctuator.
str = source.Substring(index, 4);
if (str == ">>>=")
{
index += 4;
}
else
{
// 3-character punctuators.
str = str.Substring(0, 3);
if (str == "==" || str == "!==" || str == ">>>" ||
str == "<<=" || str == ">>=")
{
index += 3;
}
else
{
// 2-character punctuators.
str = str.Substring(0, 2);
if (str == "&&" || str == "||" || str == "==" || str == "!=" ||
str == "+=" || str == "-=" || str == "*=" || str == "/=" ||
str == "++" || str == "--" || str == "<<" || str == ">>" ||
str == "&=" || str == "|=" || str == "^=" || str == "%=" ||
str == "<=" || str == ">=" || str == "=>")
{
index += 2;
}
else
{
// 1-character punctuators.
str = source[index].ToString();
if ("<>=!+-*%&|^/".IndexOf(str) >= 0)
{
++index;
}
}
}
}
break;
}
if (index == token.start)
{
throwUnexpectedToken();
}
token.end = index;
token.value = str;
return token;
}
// This function is to try to parse a MethodDefinition as defined in 14.3. But in the case of object literals,
// it might be called at a position where there is in fact a short hand identifier pattern or a data property.
// This can only be determined after we consumed up to the left parentheses.
//
// In order to avoid back tracking, it returns `null` if the position is not a MethodDefinition and the caller
// is responsible to visit other options.
public Node tryParseMethodDefinition(Token token, Node key, bool computed, Node node)
{
Node value;
Options options;
Node methodNode;
Options @params;
bool previousAllowYield = state.allowYield;
if (token.type == TokenType.Identifier)
{
// check for `get` and `set`;
if (token.value == "get" && lookaheadPropertyName())
{
computed = match("[");
key = parseObjectPropertyKey();
methodNode = new Node();
expect("(");
expect(")");
state.allowYield = false;
value = parsePropertyFunction(methodNode, new Options()
{
@params = new List<Node>(),
defaults = new List<Node>(),
stricted = null,
firstRestricted = null,
message = null
}, false);
state.allowYield = previousAllowYield;
return node.finishProperty("get", key, computed, value, false, false);
}
else if (token.value == "set" && lookaheadPropertyName())
{
computed = match("[");
key = parseObjectPropertyKey();
methodNode = new Node();
expect("(");
options = new Options()
{
@params = new List<Node>(),
defaultCount = 0,
defaults = new List<Node>(),
firstRestricted = null,
// paramSet = {}
};
if (match(")"))
{
tolerateUnexpectedToken(lookahead);
}
else
{
state.allowYield = false;
parseParam(options);
state.allowYield = previousAllowYield;
if (options.defaultCount == 0)
{
options.defaults = new List<Node>();
}
}
expect(")");
state.allowYield = false;
value = parsePropertyFunction(methodNode, options, false);
state.allowYield = previousAllowYield;
return node.finishProperty("set", key, computed, value, false, false);
}
}
else if (token.type == TokenType.Punctuator && token.value == "*" && lookaheadPropertyName())
{
computed = match("[");
key = parseObjectPropertyKey();
methodNode = new Node();
state.allowYield = true;
@params = parseParams();
state.allowYield = previousAllowYield;
state.allowYield = false;
value = parsePropertyFunction(methodNode, @params, true);
state.allowYield = previousAllowYield;
return node.finishProperty("init", key, computed, value, true, false);
}
if (key != null && match("("))
{
value = parsePropertyMethodFunction();
return node.finishProperty("init", key, computed, value, true, false);
}
// Not a MethodDefinition.
return null;
}
private bool IsIdentifierName(Token token)
{
return token.Type == TokenType.Identifier ||
token.Type == TokenType.Keyword ||
token.Type == TokenType.BooleanLiteral ||
token.Type == TokenType.NullLiteral;
}
public Node parseObjectProperty( bool hasProto)
{
var token = lookahead;
Node node = new Node();
bool computed;
Node key = null;
Node maybeMethod;
bool proto;
Node value;
computed = match("[");
if (match("*")) {
lex();
} else {
key = parseObjectPropertyKey();
}
maybeMethod = tryParseMethodDefinition(token, key, computed, node);
if (maybeMethod!= null) {
return maybeMethod;
}
if (key== null) {
throwUnexpectedToken(lookahead);
}
// Check for duplicated __proto__
if (!computed) {
proto = (key.type == Syntax.Identifier && key.name == "__proto__") ||
(key.type == Syntax.Literal && key.name== "__proto__");
if (hasProto && proto) {
tolerateError(Messages.DuplicateProtoProperty);
}
hasProto |= proto;
}
if (match(":")) {
lex();
value = inheritCoverGrammar(parseAssignmentExpression);
return node.finishProperty("init", key, computed, value, false, false);
}
if (token.type == TokenType.Identifier) {
if (match("=")) {
firstCoverInitializedNameError = lookahead;
lex();
value = isolateCoverGrammar(parseAssignmentExpression);
return node.finishProperty("init", key, computed,
new Node(token).finishAssignmentPattern(key, value), false, true);
}
return node.finishProperty("init", key, computed, key, false, true);
}
throwUnexpectedToken(lookahead);
return null;
}
private Token Lookahead()
{
if (_buffer != null)
{
return _buffer;
}
var pos = _index;
var line = _lineNumber;
var start = _lineStart;
_buffer = Advance();
_index = pos;
_lineNumber = line;
_lineStart = start;
return _buffer;
}
public int binaryPrecedence(Token token, bool allowIn)
{
var prec = 0;
if (token.type != TokenType.Punctuator && token.type != TokenType.Keyword)
{
return 0;
}
switch (token.value.ToString())
{
case "||":
prec = 1;
break;
case "&&":
prec = 2;
break;
case "|":
prec = 3;
break;
case "^":
prec = 4;
break;
case "&":
prec = 5;
break;
case "==":
case "!=":
//case "==":
case "!==":
prec = 6;
break;
case "<":
case ">":
case "<=":
case ">=":
case "instanceof":
prec = 7;
break;
case "in":
prec = allowIn ? 7 : 0;
break;
case "<<":
case ">>":
case ">>>":
prec = 8;
break;
case "+":
case "-":
prec = 9;
break;
case "*":
case "/":
case "%":
prec = 11;
break;
default:
break;
}
return prec;
}
private void ThrowErrorTolerant(Token token, string msg)
{
try
{
ThrowError(token, msg);
}
catch (Exception e)
{
if (_extra.Errors.Count > 0)
{
_extra.Errors.Add(e);
}
else
{
throw e;
}
}
}
// ECMA-262 12.14 Assignment Operators
public Node parseAssignmentExpression()
{
Token token;
Node expr;
Node right;
Options list;
Token startToken;
startToken = lookahead;
token = lookahead;
if (!state.allowYield && matchKeyword("yield"))
{
return parseYieldExpression();
}
expr = parseConditionalExpression();
if (expr.type == PlaceHolders.ArrowParameterPlaceHolder || match("=>"))
{
isAssignmentTarget = isBindingElement = false;
list = reinterpretAsCoverFormalsList(expr);
if (list != null)
{
firstCoverInitializedNameError = null;
return parseArrowFunctionExpression(list, new Node(startToken));
}
return expr;
}
if (matchAssign())
{
if (!isAssignmentTarget)
{
tolerateError(Messages.InvalidLHSInAssignment);
}
// ECMA-262 12.1.1
if (strict && expr.type == Syntax.Identifier)
{
if (isRestrictedWord(expr.name))
{
tolerateUnexpectedToken(token, Messages.StrictLHSAssignment);
}
if (isStrictModeReservedWord(expr.name))
{
tolerateUnexpectedToken(token, Messages.StrictReservedWord);
}
}
if (!match("="))
{
isAssignmentTarget = isBindingElement = false;
}
else
{
reinterpretExpressionAsPattern(expr);
}
token = lex();
right = isolateCoverGrammar(parseAssignmentExpression);
expr = new Node(startToken).finishAssignmentExpression(token.value, expr, right);
firstCoverInitializedNameError = null;
}
return expr;
}
// 11.1.5 Object Initialiser
private dynamic ParsePropertyFunction(List<Parameter> param, Token first)
{
//var previousStrict, body;
var previousStrict = _strict;
var body = ParseFunctionSourceElements();
if (first != null && _strict && IsRestrictedWord(param[0].Name))
{
ThrowError(first, Messages.StrictParamName);
}
_strict = previousStrict;
return new
{
Type = Syntax.FunctionExpression,
//Id = null,
Params = param,
//Defaults = null,
Body = body,
//Rest = null,
Generator = false,
IsExpression = false
};
}
public Options parseParams(Token firstRestricted = null)
{
Options options;
options = new Options
{
@params = new List<Node>(),
defaultCount = 0,
defaults = new List<Node>(),
firstRestricted = firstRestricted
};
expect("(");
if (!match(")"))
{
//options.paramSet =
//{
//}
//;
while (startIndex < length)
{
if (!parseParam(options))
{
break;
}
expect(",");
}
}
expect(")");
if (options.defaultCount == 0)
{
options.defaults = new List<Node>();
}
return new Options()
{
@params = options.@params,
defaults = options.defaults,
stricted = options.stricted,
firstRestricted = options.firstRestricted,
message = options.message
};
}
private dynamic CreateLiteral(Token token)
{
return new Literal(_codeGeneration)
{
IsString = token.Type == TokenType.StringLiteral,
Value = token.Value,
Range = new Range { Start = token.Range.Start, End = token.Range.End },
Loc =
new Loc
{
Start = new Loc.Position { Line = token.Loc.Start.Line, Column = token.Loc.Start.Column },
End = new Loc.Position { Line = token.Loc.End.Line, Column = token.Loc.End.Column }
}
};
}
public List<Token> tokenize(string code, Options options)
{
//var toString;
List<Token> tokens;
//toString = String;
//if (typeof code !== "string" && !(code instanceof String)) {
// code = toString(code);
//}
source = code;
index = 0;
lineNumber = (source.Length > 0) ? 1 : 0;
lineStart = 0;
startIndex = index;
startLineNumber = lineNumber;
startLineStart = lineStart;
length = source.Length;
lookahead = null;
state = new State()
{
allowIn = true,
allowYield = true,
labelSet = new List<string>(),
inFunctionBody = false,
inIteration = false,
inSwitch = false,
lastCommentStart = -1,
curlyStack = new Stack<string>()
};
extra = new Extra();
// Options matching.
options = options ?? new Options();
// Of course we collect tokens here.
options.tokens = true;
extra.tokens = new List<Token>();
extra.tokenize = true;
// The following two fields are necessary to compute the Regex tokens.
extra.openParenToken = -1;
extra.openCurlyToken = -1;
extra.range = options.range;
extra.loc = options.loc;
if (options.comment)
{
extra.comments = new List<Comment>();
}
if (options.tolerant)
{
extra.errors = new List<Error>();
}
//try
//{
peek();
if (lookahead.type == TokenType.EOF)
{
return extra.tokens;
}
lex();
while (lookahead.type != TokenType.EOF)
{
try
{
lex();
}
catch (Error lexError)
{
if (extra.errors != null)
{
recordError(lexError);
// We have to break on the first error
// to avoid infinite loops.
break;
}
else
{
throw lexError;
}
}
}
filterTokenLocation();
tokens = extra.tokens;
//tokens.comments = extra.comments;
//tokens.errors = extra.errors;
//}
//catch (Exception e)
//{
// throw e;
//}
//finally
//{
// extra = new Extra();
//}
return tokens;
}
public Program Parse(ICodeGeneration codeGeneration, string code)
{
_codeGeneration = codeGeneration;
_lineNumber = (code.Length > 0) ? 1 : 0;
_lineStart = 0;
_length = code.Length;
_buffer = null;
_state = new State
{
AllowIn = true,
LabelSet = new Dictionary<string, object>(),
LastParenthesized = null,
InFunctionBody = false,
InIteration = false,
InSwitch = false
};
_extra = new Extra();
if (_length > 0)
{
_source = StringToArray(code).ToList();
}
return ParseProgram();
}
public bool isIdentifierName(Token token)
{
return token.type == TokenType.Identifier ||
token.type == TokenType.Keyword ||
token.type == TokenType.BooleanLiteral ||
token.type == TokenType.NullLiteral;
}