public void GetNextToken() { PreviousTokenType = TokenType; TokenType = LexTypes.Eof; TokenString = string.Empty; while (CurrentChar != (char)0 && CurrentChar.IsWhitespace()) { GetNextChar(); } //single line comment if (CurrentChar == '/' && NextChar == '/') { while (CurrentChar != 0 && CurrentChar != '\n') { GetNextChar(); } GetNextChar(); GetNextToken(); return; } //multi line comment if (CurrentChar == '/' && NextChar == '*') { while (CurrentChar != 0 && (CurrentChar != '*' || NextChar != '/')) { GetNextChar(); } GetNextChar(); GetNextChar(); GetNextToken(); return; } TokenStart = dataPos - 2; if (CurrentChar.IsAlpha()) //IDs { while (CurrentChar.IsAlpha() || CurrentChar.IsNumeric()) { TokenString += CurrentChar; GetNextChar(); } TokenType = LexTypes.Id; switch (TokenString) { case "if": TokenType = LexTypes.RIf; break; case "else": TokenType = LexTypes.RElse; break; case "do": TokenType = LexTypes.RDo; break; case "while": TokenType = LexTypes.RWhile; break; case "for": TokenType = LexTypes.RFor; break; case "break": TokenType = LexTypes.RBreak; break; case "continue": TokenType = LexTypes.RContinue; break; case "function": TokenType = LexTypes.RFunction; break; case "return": TokenType = LexTypes.RReturn; break; case "var": TokenType = LexTypes.RVar; break; case "true": TokenType = LexTypes.RTrue; break; case "false": TokenType = LexTypes.RFalse; break; case "null": TokenType = LexTypes.RNull; break; case "undefined": TokenType = LexTypes.RUndefined; break; case "new": TokenType = LexTypes.RNew; break; case "typeof": TokenType = LexTypes.RTypeOf; break; case "try": TokenType = LexTypes.RTry; break; case "catch": TokenType = LexTypes.RCatch; break; case "finally": TokenType = LexTypes.RFinally; break; case "throw": TokenType = LexTypes.RThrow; break; case "const": TokenType = LexTypes.RConst; break; } } else if (CurrentChar.IsNumeric()) //Numbers { var isHex = false; if (CurrentChar == '0') { TokenString += CurrentChar; GetNextChar(); } if (CurrentChar == 'x') { isHex = true; TokenString += CurrentChar; GetNextChar(); } TokenType = LexTypes.Int; while (CurrentChar.IsNumeric() || (isHex && CurrentChar.IsHexadecimal())) { TokenString += CurrentChar; GetNextChar(); } if (!isHex && CurrentChar == '.') { TokenType = LexTypes.Float; TokenString += '.'; GetNextChar(); while (CurrentChar.IsNumeric()) { TokenString += CurrentChar; GetNextChar(); } } if (!isHex && (CurrentChar == 'e' || CurrentChar == 'E')) { TokenType = LexTypes.Float; TokenString += CurrentChar; GetNextChar(); if (CurrentChar == '-') { TokenString += CurrentChar; GetNextChar(); } while (CurrentChar.IsNumeric()) { TokenString += CurrentChar; GetNextChar(); } } } else if (CurrentChar == '\'' || CurrentChar == '\"') //Strings again { var endChar = CurrentChar; GetNextChar(); while (CurrentChar != (char)0 && CurrentChar != endChar) { if (CurrentChar == '\\') { GetNextChar(); switch (CurrentChar) { case '\n': break; case 'n': TokenString += '\n'; break; case 'r': TokenString += '\r'; break; case 'a': TokenString += '\a'; break; case 'b': TokenString += '\b'; break; case 'f': TokenString += '\f'; break; case 't': TokenString += '\t'; break; case 'v': TokenString += '\v'; break; case 'x': { var str = ""; GetNextChar(); str += CurrentChar; GetNextChar(); str += CurrentChar; TokenString += (char)Convert.ToInt64(str); } break; default: if (CurrentChar >= '0' && CurrentChar <= '7') { var str = ""; str += CurrentChar; GetNextChar(); str += CurrentChar; GetNextChar(); str += CurrentChar; TokenString += (char)Convert.ToInt64(str); } else { TokenString += CurrentChar; } break; } } else { TokenString += CurrentChar; } GetNextChar(); } GetNextChar(); TokenType = LexTypes.Str; } else //Single character { TokenType = (LexTypes)CurrentChar; if (CurrentChar != (char)0) { GetNextChar(); } if (TokenType == (LexTypes)'=' && CurrentChar == '=') // == { TokenType = LexTypes.Equal; GetNextChar(); if (CurrentChar == '=') //=== { TokenType = LexTypes.TypeEqual; GetNextChar(); } } else if (TokenType == (LexTypes)'!' && CurrentChar == '=') // != { TokenType = LexTypes.NEqual; GetNextChar(); if (CurrentChar == '=') // !== { TokenType = LexTypes.NTypeEqual; GetNextChar(); } } else if (TokenType == (LexTypes)'<' && CurrentChar == '=') // <= { TokenType = LexTypes.LEqual; GetNextChar(); } else if (TokenType == (LexTypes)'<' && CurrentChar == '<') // << { TokenType = LexTypes.LShift; GetNextChar(); if (CurrentChar == '=') //<<= { TokenType = LexTypes.LShiftEqual; GetNextChar(); } } else if (TokenType == (LexTypes)'>' && CurrentChar == '=') // >= { TokenType = LexTypes.GEqual; GetNextChar(); } else if (TokenType == (LexTypes)'>' && CurrentChar == '>') // >> { TokenType = LexTypes.RShift; GetNextChar(); if (CurrentChar == '=') // >>= { TokenType = LexTypes.RShiftEqual; GetNextChar(); } else if (CurrentChar == '>') // >>> { TokenType = LexTypes.RShiftUnsigned; GetNextChar(); if (CurrentChar == '=') // >>>= { TokenType = LexTypes.RShiftUnsignedEqual; GetNextChar(); } } } else if (TokenType == (LexTypes)'+' && CurrentChar == '=') // += { TokenType = LexTypes.PlusEqual; GetNextChar(); } else if (TokenType == (LexTypes)'-' && CurrentChar == '=') // -= { TokenType = LexTypes.MinusEqual; GetNextChar(); } else if (TokenType == (LexTypes)'+' && CurrentChar == '+') // ++ { TokenType = LexTypes.PlusPlus; GetNextChar(); } else if (TokenType == (LexTypes)'-' && CurrentChar == '-') // -- { TokenType = LexTypes.MinusMinus; GetNextChar(); } else if (TokenType == (LexTypes)'&' && CurrentChar == '=') // &= { TokenType = LexTypes.AndEqual; GetNextChar(); } else if (TokenType == (LexTypes)'&' && CurrentChar == '&') // && { TokenType = LexTypes.AndAnd; GetNextChar(); } else if (TokenType == (LexTypes)'|' && CurrentChar == '=') // |= { TokenType = LexTypes.OrEqual; GetNextChar(); } else if (TokenType == (LexTypes)'|' && CurrentChar == '|') // || { TokenType = LexTypes.OrOr; GetNextChar(); } else if (TokenType == (LexTypes)'^' && CurrentChar == '=') // ^= { TokenType = LexTypes.XorEqual; GetNextChar(); } else if (TokenType == (LexTypes)'/') { //omit regex for now TokenType = LexTypes.RegExp; foreach (var item in notAllowedBeforeRegex) { if (item == PreviousTokenType) { TokenType = (LexTypes)'/'; break; } } if (TokenType == LexTypes.RegExp) { TokenString = "/"; while (CurrentChar != 0 && CurrentChar != '/' && CurrentChar != '\n') { if (CurrentChar == '\\' && NextChar == '/') { TokenString += CurrentChar; GetNextChar(); } TokenString += CurrentChar; GetNextChar(); } if (CurrentChar == '/') { var regexStr = TokenString.Substring(1); try { var regexObj = new Regex(regexStr, RegexOptions.ECMAScript); } catch (Exception ex) { throw new ScriptException("Invalid RegEx", ex); } do { TokenString += CurrentChar; GetNextChar(); } while (CurrentChar == 'g' || CurrentChar == 'i' || CurrentChar == 'm' || CurrentChar == 'y'); } else { } } else if (CurrentChar == '=') // /= { TokenType = LexTypes.SlashEqual; GetNextChar(); } } else if (TokenType == (LexTypes)'%' && CurrentChar == '=') // %= { TokenType = LexTypes.PercentEqual; GetNextChar(); } } /* Something broke... */ TokenLastEnd = TokenEnd; TokenEnd = dataPos - 3; }
public void GetNextToken() { ColumnNumber = _currColumnNumber; TokenType = LexTypes.Eof; TokenString = String.Empty; while (CurrentChar != (char)0 && CurrentChar.IsWhitespace()) { GetNextChar(); } //single line comment if (CurrentChar == '/' && NextChar == '/') { while (CurrentChar != 0 && CurrentChar != '\n') { GetNextChar(); } GetNextChar(); GetNextToken(); return; } //multi line comment if (CurrentChar == '/' && NextChar == '*') { while (CurrentChar != '*' && CurrentChar != '/') { GetNextChar(); } GetNextChar(); GetNextChar(); GetNextToken(); return; } TokenStart = _dataPos - 2; if (CurrentChar.IsAlpha()) //IDs { while (CurrentChar.IsAlpha() || CurrentChar.IsNumeric()) { TokenString += CurrentChar; GetNextChar(); } TokenType = LexTypes.Id; switch (TokenString) { case "if": TokenType = LexTypes.RIf; break; case "else": TokenType = LexTypes.RElse; break; case "do": TokenType = LexTypes.RDo; break; case "while": TokenType = LexTypes.RWhile; break; case "for": TokenType = LexTypes.RFor; break; case "break": TokenType = LexTypes.RBreak; break; case "continue": TokenType = LexTypes.RContinue; break; case "function": TokenType = LexTypes.RFunction; break; case "return": TokenType = LexTypes.RReturn; break; case "var": TokenType = LexTypes.RVar; break; case "true": TokenType = LexTypes.RTrue; break; case "false": TokenType = LexTypes.RFalse; break; case "null": TokenType = LexTypes.RFalse; break; case "undefined": TokenType = LexTypes.RNull; break; case "new": TokenType = LexTypes.RNew; break; } } else if (CurrentChar.IsNumeric()) //Numbers { bool isHex = false; if (CurrentChar == '0') { TokenString += CurrentChar; GetNextChar(); } if (CurrentChar == 'x') { isHex = true; TokenString += CurrentChar; GetNextChar(); } TokenType = LexTypes.Int; while (CurrentChar.IsNumeric() || (isHex && CurrentChar.IsHexadecimal())) { TokenString += CurrentChar; GetNextChar(); } if (!isHex && CurrentChar == '.') { TokenType = LexTypes.Float; TokenString += '.'; GetNextChar(); while (CurrentChar.IsNumeric()) { TokenString += CurrentChar; GetNextChar(); } } if (!isHex && (CurrentChar == 'e' || CurrentChar == 'E')) { TokenType = LexTypes.Float; TokenString += CurrentChar; GetNextChar(); if (CurrentChar == '-') { TokenString += CurrentChar; GetNextChar(); } while (CurrentChar.IsNumeric()) { TokenString += CurrentChar; GetNextChar(); } } } else if (CurrentChar == '"') //Strings { GetNextChar(); while (CurrentChar != (char)0 && CurrentChar != '"') { if (CurrentChar == '\\') { GetNextChar(); switch (CurrentChar) { case 'n': TokenString += '\n'; break; case '"': TokenString += '"'; break; case '\\': TokenString += '\\'; break; default: TokenString += CurrentChar; break; } } else { TokenString += CurrentChar; } GetNextChar(); } GetNextChar(); TokenType = LexTypes.Str; } else if (CurrentChar == '\'') //Strings again { GetNextChar(); while (CurrentChar != (char)0 && CurrentChar != '\'') { if (CurrentChar == '\\') { GetNextChar(); switch (CurrentChar) { case 'n': TokenString += '\n'; break; case 'a': TokenString += '\a'; break; case 'r': TokenString += '\r'; break; case 't': TokenString += '\t'; break; case '\'': TokenString += '\''; break; case '\\': TokenString += '\\'; break; case 'x': { String str = ""; GetNextChar(); str += CurrentChar; GetNextChar(); str += CurrentChar; TokenString += (char)Convert.ToInt64(str); } break; default: if (CurrentChar >= '0' && CurrentChar <= '7') { String str = ""; str += CurrentChar; GetNextChar(); str += CurrentChar; GetNextChar(); str += CurrentChar; TokenString += (char)Convert.ToInt64(str); } else { TokenString += CurrentChar; } break; } } else { TokenString += CurrentChar; } GetNextChar(); } GetNextChar(); TokenType = LexTypes.Str; } else //Single character { TokenType = (LexTypes)CurrentChar; if (CurrentChar != (char)0) { GetNextChar(); } if (TokenType == (LexTypes)'=' && CurrentChar == '=') // == { TokenType = LexTypes.Equal; GetNextChar(); if (CurrentChar == '=') //=== { TokenType = LexTypes.TypeEqual; GetNextChar(); } } else if (TokenType == (LexTypes)'!' && CurrentChar == '=') // != { TokenType = LexTypes.NEqual; GetNextChar(); } else if (TokenType == (LexTypes)'<' && CurrentChar == '=') // <= { TokenType = LexTypes.LEqual; GetNextChar(); if (CurrentChar == '=') //!== { TokenType = LexTypes.NTypeEqual; GetNextChar(); } } else if (TokenType == (LexTypes)'<' && CurrentChar == '<') // << { TokenType = LexTypes.LShift; GetNextChar(); if (CurrentChar == '=') //<<= { TokenType = LexTypes.LShiftEqual; GetNextChar(); } } else if (TokenType == (LexTypes)'>' && CurrentChar == '=') // >= { TokenType = LexTypes.GEqual; GetNextChar(); } else if (TokenType == (LexTypes)'>' && CurrentChar == '>') // >> { TokenType = LexTypes.RShift; GetNextChar(); if (CurrentChar == '=') // >>= { TokenType = LexTypes.RShiftEqual; GetNextChar(); } else if (CurrentChar == '>') // >>> { TokenType = LexTypes.RShiftUnsigned; GetNextChar(); } } else if (TokenType == (LexTypes)'+' && CurrentChar == '=') // += { TokenType = LexTypes.PlusEqual; GetNextChar(); } else if (TokenType == (LexTypes)'-' && CurrentChar == '=') // -= { TokenType = LexTypes.MinusEqual; GetNextChar(); } else if (TokenType == (LexTypes)'+' && CurrentChar == '+') // ++ { TokenType = LexTypes.PlusPlus; GetNextChar(); } else if (TokenType == (LexTypes)'-' && CurrentChar == '-') // -- { TokenType = LexTypes.MinusMinus; GetNextChar(); } else if (TokenType == (LexTypes)'&' && CurrentChar == '=') // &= { TokenType = LexTypes.AndEqual; GetNextChar(); } else if (TokenType == (LexTypes)'&' && CurrentChar == '&') // && { TokenType = LexTypes.AndAnd; GetNextChar(); } else if (TokenType == (LexTypes)'|' && CurrentChar == '=') // |= { TokenType = LexTypes.OrEqual; GetNextChar(); } else if (TokenType == (LexTypes)'|' && CurrentChar == '|') // || { TokenType = LexTypes.OrOr; GetNextChar(); } else if (TokenType == (LexTypes)'^' && CurrentChar == '=') // ^= { TokenType = LexTypes.XorEqual; GetNextChar(); } } /* Something broke... */ TokenLastEnd = TokenEnd; TokenEnd = _dataPos - 3; }