public TokenType get() { Token token; while (this.lookahead != 0) { --this.lookahead; this.tokenIndex = (this.tokenIndex + 1) & 3; token = this.tokens[this.tokenIndex]; if (token.type != TokenType.NEWLINE || this.scanNewlines) { return(token.type); } } string capInput; Match capMatch; for (;;) { capInput = this.input; capMatch = (this.scanNewlines ? snRegExp : nnRegExp).Match(capInput); if (capMatch.Success) { string spaces = capMatch.Value; this.cursor += spaces.Length; Match nlMatch = elRegExp.Match(spaces); if (nlMatch.Value != "") { this.lineno += nlMatch.Length; } capInput = this.input; } if (!(capMatch = cmRegExp.Match(capInput)).Success) { break; } string comment = capMatch.Value; this.cursor += comment.Length; string newlines = elRegExp.Match(comment).Value; if (newlines != "") { this.lineno += newlines.Length; } } this.tokenIndex = (this.tokenIndex + 1) & 3; if (!this.tokens.TryGetValue(this.tokenIndex, out token)) { token = null; } if (token == null) { token = new Token(); this.tokens[this.tokenIndex] = token; } if (capInput == "") { return(token.type = TokenType.END); } if ((capMatch = fpRegExp.Match(capInput)).Success) { token.type = TokenType.NUMBER; token.value = JSObject.ToNumber(GLOBAL, capMatch.Groups[0].Value); } else if ((capMatch = hxRegExp.Match(capInput)).Success) { token.type = TokenType.NUMBER; token.value = JSObject.ToNumber(GLOBAL, capMatch.Groups[0].Value); } else if ((capMatch = idRegExp.Match(capInput)).Success) // FIXME no ES3 unicode { string id = capMatch.Groups[0].Value; if (!jsdefs.keywords.TryGetValue(id, out token.type)) { token.type = TokenType.IDENTIFIER; } token.value = id; } else if ((capMatch = qtRegExp.Match(capInput)).Success) //"){ { char matchCh = capMatch.Groups[0].Value[0]; int matched; token.type = TokenType.STRING; token.value = JSObject.StringLiteral(capInput, out matched); capMatch = new Regex("^" + matchCh + ".{" + matched.ToString() + "}" + matchCh).Match(capInput); } else if (this.scanOperand && (capMatch = reRegExp.Match(capInput)).Success) { token.type = TokenType.REGEXP; token.value = new Regex(capMatch.Groups[1].Value); //, capMatch.Groups[2].Value); } else if ((capMatch = opRegExp.Match(capInput)).Success) { string op = capMatch.Groups[0].Value; if (jsdefs.assignOps.ContainsKey(op) && capInput[op.Length] == '=') { token.type = TokenType.ASSIGN; token.assignOp = jsdefs.assignOps[op]; capMatch = dtRegExp.Match(op + "="); } else { token.type = jsdefs.tokenWords[op]; if (this.scanOperand && (token.type == TokenType.PLUS)) { token.type = TokenType.UNARY_PLUS; } if (this.scanOperand && (token.type == TokenType.MINUS)) { token.type = TokenType.UNARY_MINUS; } token.assignOp = TokenType.NULL; } token.value = op; } else if (this.scanNewlines && (capMatch = (elRegExp.Match(capInput))).Success) { token.type = TokenType.NEWLINE; } else { throw this.newSyntaxError("Illegal token"); } token.start = this.cursor; this.cursor += capMatch.Groups[0].Value.Length; token.end = this.cursor; token.lineno = this.lineno; return(token.type); }
public TokenType get() { Token t; while (m_lookAhead != 0) { --m_lookAhead; m_tokenIndex = (m_tokenIndex + 1) % m_tokens.Length; t = m_tokens[m_tokenIndex]; if (t.type != TokenType.NEWLINE || m_scanNewLines) { return(t.type); } } // skip over whitespace, comments char cur, next; while (m_cursor < m_source.Length) { cur = m_source[m_cursor]; // space, tab if (cur == ' ' || cur == '\t') { ++m_cursor; continue; } // all whitespace (unless m_scanNewLines on) if (!m_scanNewLines) { if (cur < ' ') // whitespace effectively... { if (cur == '\n') { ++m_lineNr; } ++m_cursor; continue; } } // comments? if (cur != '/' || m_cursor + 1 == m_source.Length) { break; } next = m_source[m_cursor + 1]; if (next == '/') { // single line comment while (++m_cursor < m_source.Length) { if (m_source[m_cursor] == '\n') { break; } } } else if (next == '*') { // multi line comment while (++m_cursor < m_source.Length - 1) { cur = m_source[m_cursor]; if (cur == '\n') { ++m_lineNr; continue; } if (cur == '*' && m_source[m_cursor + 1] == '/') { m_cursor += 2; break; } } } else { // not a comment, done break; } } m_tokenIndex = (m_tokenIndex + 1) % m_tokens.Length; t = m_tokens[m_tokenIndex]; t.start = m_cursor; if (m_cursor == m_source.Length) { t.type = TokenType.END; return(TokenType.END); } cur = m_source[m_cursor]; next = m_cursor + 1 < m_source.Length?m_source[m_cursor + 1]:' '; if ((cur >= '0' && cur <= '9') || (cur == '.' && next >= '0' && next <= '9')) { t.type = TokenType.NUMBER; if (++m_cursor < m_source.Length) { if (next == 'x' || next == 'X') { while (++m_cursor < m_source.Length) { next = m_source[m_cursor]; if (next < '0' || (next > '9' && next < 'A') || (next > 'F' && next < 'a') || next > 'f') { break; } } t.value = (double)Convert.ToInt64(m_source.Substring(t.start + 2, m_cursor - (t.start + 2)), 16); } else if (cur == '0' && next >= '0' && next < '7') { while (++m_cursor < m_source.Length) { next = m_source[m_cursor]; if (next < '0' || next > '7') { break; } } t.value = (double)Convert.ToInt64(m_source.Substring(t.start, m_cursor - t.start), 8); } else { --m_cursor; while (++m_cursor < m_source.Length) { next = m_source[m_cursor]; if (next >= '0' && next <= '9') { continue; } if (next == '.') { continue; } if (next == 'e' || next == 'E') { next = m_source[m_cursor]; if (next == '+' || next == '-') { ++m_cursor; } continue; } break; } t.value = double.Parse(m_source.Substring(t.start, m_cursor - t.start)); } } } else if (cur == '$' || cur == '_' || (cur >= 'a' && cur <= 'z') || (cur >= 'A' && cur <= 'Z')) { while (++m_cursor < m_source.Length) { next = m_source[m_cursor]; if (next == '$' || next == '_') { continue; } if (next >= 'a' && next <= 'z') { continue; } if (next >= 'A' && next <= 'Z') { continue; } if (next >= '0' && next <= '9') { continue; } break; } string s = m_source.Substring(t.start, m_cursor - t.start); t.value = s; if (!jsdefs.keywords.TryGetValue(s, out t.type)) { t.type = TokenType.IDENTIFIER; } } else if (cur == '\'' || cur == '"') { t.type = TokenType.STRING; int matched; t.value = JSObject.StringLiteral(m_source.Substring(t.start), out matched); m_cursor += matched + 2; } else { if (m_scanOperand && cur == '/') { // A regexp to match regexp literals. Regex reRegExp = new Regex("^\\/((?:\\.|\\[(?:\\.|[^\\]])*\\]|[^\\/])+)\\/([gimy]*)"); Match capMatch; if (this.scanOperand && (capMatch = reRegExp.Match(m_source.Substring(m_cursor))).Success) { t.type = TokenType.REGEXP; t.value = new Regex(capMatch.Groups[1].Value); m_cursor += capMatch.Groups [0].Value.Length; t.end = m_cursor; t.lineno = m_lineNr; return(t.type); } } t.assignOp = TokenType.NULL; ++m_cursor; switch (cur) { case '\n': t.type = TokenType.NEWLINE; break; case ';': t.type = TokenType.SEMICOLON; break; case ',': t.type = TokenType.COMMA; break; case '?': t.type = TokenType.HOOK; break; case ':': t.type = TokenType.COLON; break; case '~': t.type = TokenType.BITWISE_NOT; break; case '.': t.type = TokenType.DOT; break; case '[': t.type = TokenType.LEFT_BRACKET; break; case ']': t.type = TokenType.RIGHT_BRACKET; break; case '{': t.type = TokenType.LEFT_CURLY; break; case '}': t.type = TokenType.RIGHT_CURLY; break; case '(': t.type = TokenType.LEFT_PAREN; break; case ')': t.type = TokenType.RIGHT_PAREN; break; case '*': t.type = TokenType.MUL; break; case '/': t.type = TokenType.DIV; break; case '%': t.type = TokenType.MOD; break; case '^': t.type = TokenType.BITWISE_XOR; break; case '|': if (m_cursor < m_source.Length && m_source[m_cursor] == '|') { t.type = TokenType.OR; ++m_cursor; } else { t.type = TokenType.BITWISE_OR; } break; case '&': if (m_cursor < m_source.Length && m_source[m_cursor] == '&') { t.type = TokenType.AND; ++m_cursor; } else { t.type = TokenType.BITWISE_AND; } break; case '=': if (m_source.Substring(m_cursor, 2) == "==") { t.type = TokenType.STRICT_EQ; m_cursor += 2; } else if (m_source.Substring(m_cursor, 1) == "=") { t.type = TokenType.EQ; m_cursor += 1; } else { t.type = TokenType.ASSIGN; } break; case '!': if (m_source.Substring(m_cursor, 2) == "==") { t.type = TokenType.STRICT_NE; m_cursor += 2; } else if (m_source.Substring(m_cursor, 1) == "=") { t.type = TokenType.NE; m_cursor += 1; } else { t.type = TokenType.NOT; } break; case '+': if (m_cursor < m_source.Length && m_source[m_cursor] == '+') { t.type = TokenType.INCREMENT; ++m_cursor; } else { t.type = TokenType.PLUS; } break; case '-': if (m_cursor < m_source.Length && m_source[m_cursor] == '-') { t.type = TokenType.DECREMENT; ++m_cursor; } else { t.type = TokenType.MINUS; } break; case '<': if (m_source.Substring(m_cursor, 1) == "<") { t.type = TokenType.LSH; m_cursor += 1; } else if (m_source.Substring(m_cursor, 1) == "=") { t.type = TokenType.LE; m_cursor += 1; } else { t.type = TokenType.LT; } break; case '>': if (m_source.Substring(m_cursor, 2) == ">>") { t.type = TokenType.URSH; m_cursor += 2; } else if (m_source.Substring(m_cursor, 1) == ">") { t.type = TokenType.RSH; m_cursor += 1; } else if (m_source.Substring(m_cursor, 1) == "=") { t.type = TokenType.GE; m_cursor += 1; } else { t.type = TokenType.GT; } break; default: throw newSyntaxError("Illegal token"); } t.value = m_source.Substring(t.start, m_cursor - t.start); if (m_cursor < m_source.Length && m_source[m_cursor] == '=') { switch (t.type) { case TokenType.BITWISE_OR: case TokenType.BITWISE_XOR: case TokenType.BITWISE_AND: case TokenType.LSH: case TokenType.RSH: case TokenType.URSH: case TokenType.PLUS: case TokenType.MINUS: case TokenType.MUL: case TokenType.DIV: case TokenType.MOD: t.assignOp = t.type; t.type = TokenType.ASSIGN; ++m_cursor; break; } } if (m_scanOperand) { switch (t.type) { case TokenType.PLUS: t.type = TokenType.UNARY_PLUS; break; case TokenType.MINUS: t.type = TokenType.UNARY_MINUS; break; } } } t.end = m_cursor; t.lineno = m_lineNr; return(t.type); }