public TokenType get()
{
Token token;
while (this.lookahead != 0)
{
--this.lookahead;
this.tokenIndex = (this.tokenIndex + 1) & 3;
token = this.tokens[this.tokenIndex];
if (token.type != TokenType.NEWLINE || this.scanNewlines)
{
return(token.type);
}
}
string capInput;
Match capMatch;
for (;;)
{
capInput = this.input;
capMatch = (this.scanNewlines ? snRegExp : nnRegExp).Match(capInput);
if (capMatch.Success)
{
string spaces = capMatch.Value;
this.cursor += spaces.Length;
Match nlMatch = elRegExp.Match(spaces);
if (nlMatch.Value != "")
{
this.lineno += nlMatch.Length;
}
capInput = this.input;
}
if (!(capMatch = cmRegExp.Match(capInput)).Success)
{
break;
}
string comment = capMatch.Value;
this.cursor += comment.Length;
string newlines = elRegExp.Match(comment).Value;
if (newlines != "")
{
this.lineno += newlines.Length;
}
}
this.tokenIndex = (this.tokenIndex + 1) & 3;
if (!this.tokens.TryGetValue(this.tokenIndex, out token))
{
token = null;
}
if (token == null)
{
token = new Token();
this.tokens[this.tokenIndex] = token;
}
if (capInput == "")
{
return(token.type = TokenType.END);
}
if ((capMatch = fpRegExp.Match(capInput)).Success)
{
token.type = TokenType.NUMBER;
token.value = JSObject.ToNumber(GLOBAL, capMatch.Groups[0].Value);
}
else if ((capMatch = hxRegExp.Match(capInput)).Success)
{
token.type = TokenType.NUMBER;
token.value = JSObject.ToNumber(GLOBAL, capMatch.Groups[0].Value);
}
else if ((capMatch = idRegExp.Match(capInput)).Success) // FIXME no ES3 unicode
{
string id = capMatch.Groups[0].Value;
if (!jsdefs.keywords.TryGetValue(id, out token.type))
{
token.type = TokenType.IDENTIFIER;
}
token.value = id;
}
else if ((capMatch = qtRegExp.Match(capInput)).Success) //"){
{
char matchCh = capMatch.Groups[0].Value[0];
int matched;
token.type = TokenType.STRING;
token.value = JSObject.StringLiteral(capInput, out matched);
capMatch = new Regex("^" + matchCh + ".{" + matched.ToString() + "}" + matchCh).Match(capInput);
}
else if (this.scanOperand && (capMatch = reRegExp.Match(capInput)).Success)
{
token.type = TokenType.REGEXP;
token.value = new Regex(capMatch.Groups[1].Value); //, capMatch.Groups[2].Value);
}
else if ((capMatch = opRegExp.Match(capInput)).Success)
{
string op = capMatch.Groups[0].Value;
if (jsdefs.assignOps.ContainsKey(op) && capInput[op.Length] == '=')
{
token.type = TokenType.ASSIGN;
token.assignOp = jsdefs.assignOps[op];
capMatch = dtRegExp.Match(op + "=");
}
else
{
token.type = jsdefs.tokenWords[op];
if (this.scanOperand &&
(token.type == TokenType.PLUS))
{
token.type = TokenType.UNARY_PLUS;
}
if (this.scanOperand &&
(token.type == TokenType.MINUS))
{
token.type = TokenType.UNARY_MINUS;
}
token.assignOp = TokenType.NULL;
}
token.value = op;
}
else if (this.scanNewlines && (capMatch = (elRegExp.Match(capInput))).Success)
{
token.type = TokenType.NEWLINE;
}
else
{
throw this.newSyntaxError("Illegal token");
}
token.start = this.cursor;
this.cursor += capMatch.Groups[0].Value.Length;
token.end = this.cursor;
token.lineno = this.lineno;
return(token.type);
}
public TokenType get()
{
Token t;
while (m_lookAhead != 0)
{
--m_lookAhead;
m_tokenIndex = (m_tokenIndex + 1) % m_tokens.Length;
t = m_tokens[m_tokenIndex];
if (t.type != TokenType.NEWLINE || m_scanNewLines)
{
return(t.type);
}
}
// skip over whitespace, comments
char cur, next;
while (m_cursor < m_source.Length)
{
cur = m_source[m_cursor];
// space, tab
if (cur == ' ' || cur == '\t')
{
++m_cursor;
continue;
}
// all whitespace (unless m_scanNewLines on)
if (!m_scanNewLines)
{
if (cur < ' ') // whitespace effectively...
{
if (cur == '\n')
{
++m_lineNr;
}
++m_cursor;
continue;
}
}
// comments?
if (cur != '/' || m_cursor + 1 == m_source.Length)
{
break;
}
next = m_source[m_cursor + 1];
if (next == '/')
{
// single line comment
while (++m_cursor < m_source.Length)
{
if (m_source[m_cursor] == '\n')
{
break;
}
}
}
else if (next == '*')
{
// multi line comment
while (++m_cursor < m_source.Length - 1)
{
cur = m_source[m_cursor];
if (cur == '\n')
{
++m_lineNr;
continue;
}
if (cur == '*' && m_source[m_cursor + 1] == '/')
{
m_cursor += 2;
break;
}
}
}
else
{
// not a comment, done
break;
}
}
m_tokenIndex = (m_tokenIndex + 1) % m_tokens.Length;
t = m_tokens[m_tokenIndex];
t.start = m_cursor;
if (m_cursor == m_source.Length)
{
t.type = TokenType.END;
return(TokenType.END);
}
cur = m_source[m_cursor];
next = m_cursor + 1 < m_source.Length?m_source[m_cursor + 1]:' ';
if ((cur >= '0' && cur <= '9') || (cur == '.' && next >= '0' && next <= '9'))
{
t.type = TokenType.NUMBER;
if (++m_cursor < m_source.Length)
{
if (next == 'x' || next == 'X')
{
while (++m_cursor < m_source.Length)
{
next = m_source[m_cursor];
if (next < '0' || (next > '9' && next < 'A') || (next > 'F' && next < 'a') || next > 'f')
{
break;
}
}
t.value = (double)Convert.ToInt64(m_source.Substring(t.start + 2, m_cursor - (t.start + 2)), 16);
}
else if (cur == '0' && next >= '0' && next < '7')
{
while (++m_cursor < m_source.Length)
{
next = m_source[m_cursor];
if (next < '0' || next > '7')
{
break;
}
}
t.value = (double)Convert.ToInt64(m_source.Substring(t.start, m_cursor - t.start), 8);
}
else
{
--m_cursor;
while (++m_cursor < m_source.Length)
{
next = m_source[m_cursor];
if (next >= '0' && next <= '9')
{
continue;
}
if (next == '.')
{
continue;
}
if (next == 'e' || next == 'E')
{
next = m_source[m_cursor];
if (next == '+' || next == '-')
{
++m_cursor;
}
continue;
}
break;
}
t.value = double.Parse(m_source.Substring(t.start, m_cursor - t.start));
}
}
}
else if (cur == '$' || cur == '_' || (cur >= 'a' && cur <= 'z') || (cur >= 'A' && cur <= 'Z'))
{
while (++m_cursor < m_source.Length)
{
next = m_source[m_cursor];
if (next == '$' || next == '_')
{
continue;
}
if (next >= 'a' && next <= 'z')
{
continue;
}
if (next >= 'A' && next <= 'Z')
{
continue;
}
if (next >= '0' && next <= '9')
{
continue;
}
break;
}
string s = m_source.Substring(t.start, m_cursor - t.start);
t.value = s;
if (!jsdefs.keywords.TryGetValue(s, out t.type))
{
t.type = TokenType.IDENTIFIER;
}
}
else if (cur == '\'' || cur == '"')
{
t.type = TokenType.STRING;
int matched;
t.value = JSObject.StringLiteral(m_source.Substring(t.start), out matched);
m_cursor += matched + 2;
}
else
{
if (m_scanOperand && cur == '/')
{
// A regexp to match regexp literals.
Regex reRegExp = new Regex("^\\/((?:\\.|\\[(?:\\.|[^\\]])*\\]|[^\\/])+)\\/([gimy]*)");
Match capMatch;
if (this.scanOperand && (capMatch = reRegExp.Match(m_source.Substring(m_cursor))).Success)
{
t.type = TokenType.REGEXP;
t.value = new Regex(capMatch.Groups[1].Value);
m_cursor += capMatch.Groups [0].Value.Length;
t.end = m_cursor;
t.lineno = m_lineNr;
return(t.type);
}
}
t.assignOp = TokenType.NULL;
++m_cursor;
switch (cur)
{
case '\n': t.type = TokenType.NEWLINE; break;
case ';': t.type = TokenType.SEMICOLON; break;
case ',': t.type = TokenType.COMMA; break;
case '?': t.type = TokenType.HOOK; break;
case ':': t.type = TokenType.COLON; break;
case '~': t.type = TokenType.BITWISE_NOT; break;
case '.': t.type = TokenType.DOT; break;
case '[': t.type = TokenType.LEFT_BRACKET; break;
case ']': t.type = TokenType.RIGHT_BRACKET; break;
case '{': t.type = TokenType.LEFT_CURLY; break;
case '}': t.type = TokenType.RIGHT_CURLY; break;
case '(': t.type = TokenType.LEFT_PAREN; break;
case ')': t.type = TokenType.RIGHT_PAREN; break;
case '*': t.type = TokenType.MUL; break;
case '/': t.type = TokenType.DIV; break;
case '%': t.type = TokenType.MOD; break;
case '^': t.type = TokenType.BITWISE_XOR; break;
case '|':
if (m_cursor < m_source.Length && m_source[m_cursor] == '|')
{
t.type = TokenType.OR;
++m_cursor;
}
else
{
t.type = TokenType.BITWISE_OR;
}
break;
case '&':
if (m_cursor < m_source.Length && m_source[m_cursor] == '&')
{
t.type = TokenType.AND;
++m_cursor;
}
else
{
t.type = TokenType.BITWISE_AND;
}
break;
case '=':
if (m_source.Substring(m_cursor, 2) == "==")
{
t.type = TokenType.STRICT_EQ;
m_cursor += 2;
}
else if (m_source.Substring(m_cursor, 1) == "=")
{
t.type = TokenType.EQ;
m_cursor += 1;
}
else
{
t.type = TokenType.ASSIGN;
}
break;
case '!':
if (m_source.Substring(m_cursor, 2) == "==")
{
t.type = TokenType.STRICT_NE;
m_cursor += 2;
}
else if (m_source.Substring(m_cursor, 1) == "=")
{
t.type = TokenType.NE;
m_cursor += 1;
}
else
{
t.type = TokenType.NOT;
}
break;
case '+':
if (m_cursor < m_source.Length && m_source[m_cursor] == '+')
{
t.type = TokenType.INCREMENT;
++m_cursor;
}
else
{
t.type = TokenType.PLUS;
}
break;
case '-':
if (m_cursor < m_source.Length && m_source[m_cursor] == '-')
{
t.type = TokenType.DECREMENT;
++m_cursor;
}
else
{
t.type = TokenType.MINUS;
}
break;
case '<':
if (m_source.Substring(m_cursor, 1) == "<")
{
t.type = TokenType.LSH;
m_cursor += 1;
}
else if (m_source.Substring(m_cursor, 1) == "=")
{
t.type = TokenType.LE;
m_cursor += 1;
}
else
{
t.type = TokenType.LT;
}
break;
case '>':
if (m_source.Substring(m_cursor, 2) == ">>")
{
t.type = TokenType.URSH;
m_cursor += 2;
}
else if (m_source.Substring(m_cursor, 1) == ">")
{
t.type = TokenType.RSH;
m_cursor += 1;
}
else if (m_source.Substring(m_cursor, 1) == "=")
{
t.type = TokenType.GE;
m_cursor += 1;
}
else
{
t.type = TokenType.GT;
}
break;
default:
throw newSyntaxError("Illegal token");
}
t.value = m_source.Substring(t.start, m_cursor - t.start);
if (m_cursor < m_source.Length && m_source[m_cursor] == '=')
{
switch (t.type)
{
case TokenType.BITWISE_OR:
case TokenType.BITWISE_XOR:
case TokenType.BITWISE_AND:
case TokenType.LSH:
case TokenType.RSH:
case TokenType.URSH:
case TokenType.PLUS:
case TokenType.MINUS:
case TokenType.MUL:
case TokenType.DIV:
case TokenType.MOD:
t.assignOp = t.type;
t.type = TokenType.ASSIGN;
++m_cursor;
break;
}
}
if (m_scanOperand)
{
switch (t.type)
{
case TokenType.PLUS: t.type = TokenType.UNARY_PLUS; break;
case TokenType.MINUS: t.type = TokenType.UNARY_MINUS; break;
}
}
}
t.end = m_cursor;
t.lineno = m_lineNr;
return(t.type);
}
