public void GetNextToken()
{
PreviousTokenType = TokenType;
TokenType = LexTypes.Eof;
TokenString = string.Empty;
while (CurrentChar != (char)0 && CurrentChar.IsWhitespace())
{
GetNextChar();
}
//single line comment
if (CurrentChar == '/' && NextChar == '/')
{
while (CurrentChar != 0 && CurrentChar != '\n')
{
GetNextChar();
}
GetNextChar();
GetNextToken();
return;
}
//multi line comment
if (CurrentChar == '/' && NextChar == '*')
{
while (CurrentChar != 0 && (CurrentChar != '*' || NextChar != '/'))
{
GetNextChar();
}
GetNextChar();
GetNextChar();
GetNextToken();
return;
}
TokenStart = dataPos - 2;
if (CurrentChar.IsAlpha()) //IDs
{
while (CurrentChar.IsAlpha() || CurrentChar.IsNumeric())
{
TokenString += CurrentChar;
GetNextChar();
}
TokenType = LexTypes.Id;
switch (TokenString)
{
case "if": TokenType = LexTypes.RIf; break;
case "else": TokenType = LexTypes.RElse; break;
case "do": TokenType = LexTypes.RDo; break;
case "while": TokenType = LexTypes.RWhile; break;
case "for": TokenType = LexTypes.RFor; break;
case "break": TokenType = LexTypes.RBreak; break;
case "continue": TokenType = LexTypes.RContinue; break;
case "function": TokenType = LexTypes.RFunction; break;
case "return": TokenType = LexTypes.RReturn; break;
case "var": TokenType = LexTypes.RVar; break;
case "true": TokenType = LexTypes.RTrue; break;
case "false": TokenType = LexTypes.RFalse; break;
case "null": TokenType = LexTypes.RNull; break;
case "undefined": TokenType = LexTypes.RUndefined; break;
case "new": TokenType = LexTypes.RNew; break;
case "typeof": TokenType = LexTypes.RTypeOf; break;
case "try": TokenType = LexTypes.RTry; break;
case "catch": TokenType = LexTypes.RCatch; break;
case "finally": TokenType = LexTypes.RFinally; break;
case "throw": TokenType = LexTypes.RThrow; break;
case "const": TokenType = LexTypes.RConst; break;
}
}
else if (CurrentChar.IsNumeric()) //Numbers
{
var isHex = false;
if (CurrentChar == '0')
{
TokenString += CurrentChar;
GetNextChar();
}
if (CurrentChar == 'x')
{
isHex = true;
TokenString += CurrentChar;
GetNextChar();
}
TokenType = LexTypes.Int;
while (CurrentChar.IsNumeric() || (isHex && CurrentChar.IsHexadecimal()))
{
TokenString += CurrentChar;
GetNextChar();
}
if (!isHex && CurrentChar == '.')
{
TokenType = LexTypes.Float;
TokenString += '.';
GetNextChar();
while (CurrentChar.IsNumeric())
{
TokenString += CurrentChar;
GetNextChar();
}
}
if (!isHex && (CurrentChar == 'e' || CurrentChar == 'E'))
{
TokenType = LexTypes.Float;
TokenString += CurrentChar;
GetNextChar();
if (CurrentChar == '-')
{
TokenString += CurrentChar;
GetNextChar();
}
while (CurrentChar.IsNumeric())
{
TokenString += CurrentChar;
GetNextChar();
}
}
}
else if (CurrentChar == '\'' || CurrentChar == '\"') //Strings again
{
var endChar = CurrentChar;
GetNextChar();
while (CurrentChar != (char)0 && CurrentChar != endChar)
{
if (CurrentChar == '\\')
{
GetNextChar();
switch (CurrentChar)
{
case '\n': break;
case 'n':
TokenString += '\n';
break;
case 'r':
TokenString += '\r';
break;
case 'a':
TokenString += '\a';
break;
case 'b':
TokenString += '\b';
break;
case 'f':
TokenString += '\f';
break;
case 't':
TokenString += '\t';
break;
case 'v':
TokenString += '\v';
break;
case 'x':
{
var str = "";
GetNextChar();
str += CurrentChar;
GetNextChar();
str += CurrentChar;
TokenString += (char)Convert.ToInt64(str);
}
break;
default:
if (CurrentChar >= '0' && CurrentChar <= '7')
{
var str = "";
str += CurrentChar;
GetNextChar();
str += CurrentChar;
GetNextChar();
str += CurrentChar;
TokenString += (char)Convert.ToInt64(str);
}
else
{
TokenString += CurrentChar;
}
break;
}
}
else
{
TokenString += CurrentChar;
}
GetNextChar();
}
GetNextChar();
TokenType = LexTypes.Str;
}
else //Single character
{
TokenType = (LexTypes)CurrentChar;
if (CurrentChar != (char)0)
{
GetNextChar();
}
if (TokenType == (LexTypes)'=' && CurrentChar == '=') // ==
{
TokenType = LexTypes.Equal;
GetNextChar();
if (CurrentChar == '=') //===
{
TokenType = LexTypes.TypeEqual;
GetNextChar();
}
}
else if (TokenType == (LexTypes)'!' && CurrentChar == '=') // !=
{
TokenType = LexTypes.NEqual;
GetNextChar();
if (CurrentChar == '=') // !==
{
TokenType = LexTypes.NTypeEqual;
GetNextChar();
}
}
else if (TokenType == (LexTypes)'<' && CurrentChar == '=') // <=
{
TokenType = LexTypes.LEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'<' && CurrentChar == '<') // <<
{
TokenType = LexTypes.LShift;
GetNextChar();
if (CurrentChar == '=') //<<=
{
TokenType = LexTypes.LShiftEqual;
GetNextChar();
}
}
else if (TokenType == (LexTypes)'>' && CurrentChar == '=') // >=
{
TokenType = LexTypes.GEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'>' && CurrentChar == '>') // >>
{
TokenType = LexTypes.RShift;
GetNextChar();
if (CurrentChar == '=') // >>=
{
TokenType = LexTypes.RShiftEqual;
GetNextChar();
}
else if (CurrentChar == '>') // >>>
{
TokenType = LexTypes.RShiftUnsigned;
GetNextChar();
if (CurrentChar == '=') // >>>=
{
TokenType = LexTypes.RShiftUnsignedEqual;
GetNextChar();
}
}
}
else if (TokenType == (LexTypes)'+' && CurrentChar == '=') // +=
{
TokenType = LexTypes.PlusEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'-' && CurrentChar == '=') // -=
{
TokenType = LexTypes.MinusEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'+' && CurrentChar == '+') // ++
{
TokenType = LexTypes.PlusPlus;
GetNextChar();
}
else if (TokenType == (LexTypes)'-' && CurrentChar == '-') // --
{
TokenType = LexTypes.MinusMinus;
GetNextChar();
}
else if (TokenType == (LexTypes)'&' && CurrentChar == '=') // &=
{
TokenType = LexTypes.AndEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'&' && CurrentChar == '&') // &&
{
TokenType = LexTypes.AndAnd;
GetNextChar();
}
else if (TokenType == (LexTypes)'|' && CurrentChar == '=') // |=
{
TokenType = LexTypes.OrEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'|' && CurrentChar == '|') // ||
{
TokenType = LexTypes.OrOr;
GetNextChar();
}
else if (TokenType == (LexTypes)'^' && CurrentChar == '=') // ^=
{
TokenType = LexTypes.XorEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'/')
{
//omit regex for now
TokenType = LexTypes.RegExp;
foreach (var item in notAllowedBeforeRegex)
{
if (item == PreviousTokenType)
{
TokenType = (LexTypes)'/';
break;
}
}
if (TokenType == LexTypes.RegExp)
{
TokenString = "/";
while (CurrentChar != 0 && CurrentChar != '/' && CurrentChar != '\n')
{
if (CurrentChar == '\\' && NextChar == '/')
{
TokenString += CurrentChar;
GetNextChar();
}
TokenString += CurrentChar;
GetNextChar();
}
if (CurrentChar == '/')
{
var regexStr = TokenString.Substring(1);
try
{
var regexObj = new Regex(regexStr, RegexOptions.ECMAScript);
}
catch (Exception ex)
{
throw new ScriptException("Invalid RegEx", ex);
}
do
{
TokenString += CurrentChar;
GetNextChar();
} while (CurrentChar == 'g' || CurrentChar == 'i' || CurrentChar == 'm' || CurrentChar == 'y');
}
else
{
}
}
else if (CurrentChar == '=') // /=
{
TokenType = LexTypes.SlashEqual;
GetNextChar();
}
}
else if (TokenType == (LexTypes)'%' && CurrentChar == '=') // %=
{
TokenType = LexTypes.PercentEqual;
GetNextChar();
}
}
/* Something broke... */
TokenLastEnd = TokenEnd;
TokenEnd = dataPos - 3;
}
public void GetNextToken()
{
ColumnNumber = _currColumnNumber;
TokenType = LexTypes.Eof;
TokenString = String.Empty;
while (CurrentChar != (char)0 && CurrentChar.IsWhitespace())
{
GetNextChar();
}
//single line comment
if (CurrentChar == '/' && NextChar == '/')
{
while (CurrentChar != 0 && CurrentChar != '\n')
{
GetNextChar();
}
GetNextChar();
GetNextToken();
return;
}
//multi line comment
if (CurrentChar == '/' && NextChar == '*')
{
while (CurrentChar != '*' && CurrentChar != '/')
{
GetNextChar();
}
GetNextChar();
GetNextChar();
GetNextToken();
return;
}
TokenStart = _dataPos - 2;
if (CurrentChar.IsAlpha()) //IDs
{
while (CurrentChar.IsAlpha() || CurrentChar.IsNumeric())
{
TokenString += CurrentChar;
GetNextChar();
}
TokenType = LexTypes.Id;
switch (TokenString)
{
case "if": TokenType = LexTypes.RIf; break;
case "else": TokenType = LexTypes.RElse; break;
case "do": TokenType = LexTypes.RDo; break;
case "while": TokenType = LexTypes.RWhile; break;
case "for": TokenType = LexTypes.RFor; break;
case "break": TokenType = LexTypes.RBreak; break;
case "continue": TokenType = LexTypes.RContinue; break;
case "function": TokenType = LexTypes.RFunction; break;
case "return": TokenType = LexTypes.RReturn; break;
case "var": TokenType = LexTypes.RVar; break;
case "true": TokenType = LexTypes.RTrue; break;
case "false": TokenType = LexTypes.RFalse; break;
case "null": TokenType = LexTypes.RFalse; break;
case "undefined": TokenType = LexTypes.RNull; break;
case "new": TokenType = LexTypes.RNew; break;
}
}
else if (CurrentChar.IsNumeric()) //Numbers
{
bool isHex = false;
if (CurrentChar == '0')
{
TokenString += CurrentChar;
GetNextChar();
}
if (CurrentChar == 'x')
{
isHex = true;
TokenString += CurrentChar;
GetNextChar();
}
TokenType = LexTypes.Int;
while (CurrentChar.IsNumeric() || (isHex && CurrentChar.IsHexadecimal()))
{
TokenString += CurrentChar;
GetNextChar();
}
if (!isHex && CurrentChar == '.')
{
TokenType = LexTypes.Float;
TokenString += '.';
GetNextChar();
while (CurrentChar.IsNumeric())
{
TokenString += CurrentChar;
GetNextChar();
}
}
if (!isHex && (CurrentChar == 'e' || CurrentChar == 'E'))
{
TokenType = LexTypes.Float;
TokenString += CurrentChar;
GetNextChar();
if (CurrentChar == '-')
{
TokenString += CurrentChar;
GetNextChar();
}
while (CurrentChar.IsNumeric())
{
TokenString += CurrentChar;
GetNextChar();
}
}
}
else if (CurrentChar == '"') //Strings
{
GetNextChar();
while (CurrentChar != (char)0 && CurrentChar != '"')
{
if (CurrentChar == '\\')
{
GetNextChar();
switch (CurrentChar)
{
case 'n':
TokenString += '\n';
break;
case '"':
TokenString += '"';
break;
case '\\':
TokenString += '\\';
break;
default:
TokenString += CurrentChar;
break;
}
}
else
{
TokenString += CurrentChar;
}
GetNextChar();
}
GetNextChar();
TokenType = LexTypes.Str;
}
else if (CurrentChar == '\'') //Strings again
{
GetNextChar();
while (CurrentChar != (char)0 && CurrentChar != '\'')
{
if (CurrentChar == '\\')
{
GetNextChar();
switch (CurrentChar)
{
case 'n':
TokenString += '\n';
break;
case 'a':
TokenString += '\a';
break;
case 'r':
TokenString += '\r';
break;
case 't':
TokenString += '\t';
break;
case '\'':
TokenString += '\'';
break;
case '\\':
TokenString += '\\';
break;
case 'x':
{
String str = "";
GetNextChar();
str += CurrentChar;
GetNextChar();
str += CurrentChar;
TokenString += (char)Convert.ToInt64(str);
}
break;
default:
if (CurrentChar >= '0' && CurrentChar <= '7')
{
String str = "";
str += CurrentChar;
GetNextChar();
str += CurrentChar;
GetNextChar();
str += CurrentChar;
TokenString += (char)Convert.ToInt64(str);
}
else
{
TokenString += CurrentChar;
}
break;
}
}
else
{
TokenString += CurrentChar;
}
GetNextChar();
}
GetNextChar();
TokenType = LexTypes.Str;
}
else //Single character
{
TokenType = (LexTypes)CurrentChar;
if (CurrentChar != (char)0)
{
GetNextChar();
}
if (TokenType == (LexTypes)'=' && CurrentChar == '=') // ==
{
TokenType = LexTypes.Equal;
GetNextChar();
if (CurrentChar == '=') //===
{
TokenType = LexTypes.TypeEqual;
GetNextChar();
}
}
else if (TokenType == (LexTypes)'!' && CurrentChar == '=') // !=
{
TokenType = LexTypes.NEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'<' && CurrentChar == '=') // <=
{
TokenType = LexTypes.LEqual;
GetNextChar();
if (CurrentChar == '=') //!==
{
TokenType = LexTypes.NTypeEqual;
GetNextChar();
}
}
else if (TokenType == (LexTypes)'<' && CurrentChar == '<') // <<
{
TokenType = LexTypes.LShift;
GetNextChar();
if (CurrentChar == '=') //<<=
{
TokenType = LexTypes.LShiftEqual;
GetNextChar();
}
}
else if (TokenType == (LexTypes)'>' && CurrentChar == '=') // >=
{
TokenType = LexTypes.GEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'>' && CurrentChar == '>') // >>
{
TokenType = LexTypes.RShift;
GetNextChar();
if (CurrentChar == '=') // >>=
{
TokenType = LexTypes.RShiftEqual;
GetNextChar();
}
else if (CurrentChar == '>') // >>>
{
TokenType = LexTypes.RShiftUnsigned;
GetNextChar();
}
}
else if (TokenType == (LexTypes)'+' && CurrentChar == '=') // +=
{
TokenType = LexTypes.PlusEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'-' && CurrentChar == '=') // -=
{
TokenType = LexTypes.MinusEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'+' && CurrentChar == '+') // ++
{
TokenType = LexTypes.PlusPlus;
GetNextChar();
}
else if (TokenType == (LexTypes)'-' && CurrentChar == '-') // --
{
TokenType = LexTypes.MinusMinus;
GetNextChar();
}
else if (TokenType == (LexTypes)'&' && CurrentChar == '=') // &=
{
TokenType = LexTypes.AndEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'&' && CurrentChar == '&') // &&
{
TokenType = LexTypes.AndAnd;
GetNextChar();
}
else if (TokenType == (LexTypes)'|' && CurrentChar == '=') // |=
{
TokenType = LexTypes.OrEqual;
GetNextChar();
}
else if (TokenType == (LexTypes)'|' && CurrentChar == '|') // ||
{
TokenType = LexTypes.OrOr;
GetNextChar();
}
else if (TokenType == (LexTypes)'^' && CurrentChar == '=') // ^=
{
TokenType = LexTypes.XorEqual;
GetNextChar();
}
}
/* Something broke... */
TokenLastEnd = TokenEnd;
TokenEnd = _dataPos - 3;
}