public bool GetToken()
{
if (this.PendingToken != null)
{
this.Token = this.PendingToken;
this.PendingToken = null;
return(true);
}
base.MovePastInlineWhitespace();
if (base.EndOfText)
{
return(false);
}
CSSToken cSSToken = new CSSToken();
if (this.Token != null)
{
cSSToken.PreviousValue = this.Token.Value;
cSSToken.PreviousType = this.Token.Type;
}
int position = base.Position;
if (base.Peek() == '\r' || base.Peek() == '\n')
{
cSSToken.Type = CSSTokenTypes.CRLF;
base.MoveAhead();
if (base.Peek() == '\r' || base.Peek() == '\n')
{
base.MoveAhead();
}
base.MovePastInlineWhitespace();
}
else if (TextParser.StringArrayContains(CSSTokenizer._symbolChars, base.Peek()))
{
cSSToken.Type = CSSTokenTypes.Symbol;
base.MoveAhead();
while (TextParser.StringArrayContains(CSSTokenizer._symbolChars, base.Peek()))
{
base.MoveAhead();
}
}
else if (base.Peek() == '/' && base.Peek(1) == '*')
{
cSSToken.Type = CSSTokenTypes.InlineComment;
base.MoveTo("*/");
base.MoveAhead(2);
}
else if (base.Peek() == '\'' || base.Peek() == '"' || base.Peek() == '(' || base.Peek() == ')')
{
cSSToken.Type = CSSTokenTypes.String;
char c = base.Peek();
if (c == '(')
{
c = ')';
}
base.MoveAhead();
while (!base.EndOfText)
{
if (base.Peek() == c)
{
base.MoveAhead();
break;
}
if (base.Peek() == '\\' && (base.Peek(1) == '\\' || base.Peek(1) == c))
{
base.MoveAhead(2);
}
else
{
base.MoveAhead();
}
}
}
else if (char.IsDigit(base.Peek()) || (base.Peek() == '.' && char.IsDigit(base.Peek(1))))
{
cSSToken.Type = CSSTokenTypes.Number;
string decimalChars = CSSTokenizer._decimalChars;
bool flag = false;
while (TextParser.StringArrayContains(decimalChars, base.Peek()))
{
if (base.Peek() == '.')
{
if (flag)
{
break;
}
flag = true;
}
base.MoveAhead();
}
}
else if (TextParser.StringArrayContains(CSSTokenizer._operatorChars, base.Peek()))
{
char c2 = base.Peek();
bool flag2 = false;
CSSTokenizer.NamedOperator[] named = CSSTokenizer._named;
for (int i = 0; i < named.Length; i++)
{
CSSTokenizer.NamedOperator namedOperator = named[i];
if (namedOperator.Operator == c2)
{
cSSToken.Type = namedOperator.Type;
base.MoveAhead();
flag2 = true;
break;
}
}
if (!flag2)
{
if (c2 == '+' || c2 == '-')
{
cSSToken.Type = CSSTokenTypes.UnaryPrefix;
}
else
{
cSSToken.Type = CSSTokenTypes.Unknown;
}
base.MoveAhead();
}
}
else
{
cSSToken.Type = CSSTokenTypes.Unknown;
base.MoveAhead();
}
cSSToken.Value = base.Extract(position, base.Position);
this.Token = cSSToken;
return(true);
}
public bool GetToken()
{
if (this.PendingToken != null)
{
this.Token = this.PendingToken;
this.PendingToken = null;
return(true);
}
base.MovePastInlineWhitespace();
if (base.EndOfText)
{
return(false);
}
JSToken jSToken = new JSToken();
if (this.Token != null)
{
jSToken.PreviousValue = this.Token.Value;
jSToken.PreviousType = this.Token.Type;
}
int position = base.Position;
if (base.Peek() == '\r' || base.Peek() == '\n')
{
jSToken.Type = JSTokenTypes.CRLF;
base.MoveAhead();
if (base.Peek() == '\r' || base.Peek() == '\n')
{
base.MoveAhead();
}
base.MovePastInlineWhitespace();
}
else if (TextParser.StringArrayContains(JSTokenizer._firstSymbolChars, base.Peek()))
{
jSToken.Type = JSTokenTypes.Symbol;
base.MoveAhead();
while (TextParser.StringArrayContains(JSTokenizer._symbolChars, base.Peek()))
{
base.MoveAhead();
}
}
else if (base.Peek() == '/' && base.Peek(1) == '/')
{
jSToken.Type = JSTokenTypes.FullLineComment;
base.MoveToEndOfLine();
}
else if (base.Peek() == '<' && base.Peek(1) == '!' && base.Peek(2) == '-' && base.Peek(3) == '-')
{
jSToken.Type = JSTokenTypes.FullLineComment;
base.MoveToEndOfLine();
}
else if (base.Peek() == '/' && base.Peek(1) == '*')
{
jSToken.Type = JSTokenTypes.InlineComment;
base.MoveTo("*/");
base.MoveAhead(2);
}
else if (base.Peek() == '\'' || base.Peek() == '"')
{
jSToken.Type = JSTokenTypes.String;
char c = base.Peek();
base.MoveAhead();
while (!base.EndOfText)
{
if (base.Peek() == c)
{
base.MoveAhead();
break;
}
if (base.Peek() == '\\' && (base.Peek(1) == '\\' || base.Peek(1) == c))
{
base.MoveAhead(2);
}
else
{
base.MoveAhead();
}
}
}
else if (char.IsDigit(base.Peek()) || (base.Peek() == '.' && char.IsDigit(base.Peek(1))))
{
jSToken.Type = JSTokenTypes.Number;
string text = JSTokenizer._decimalChars;
if (base.Peek() == '0')
{
if (char.IsDigit(base.Peek(1)))
{
text = JSTokenizer._octalChars;
}
else if (char.ToLower(base.Peek(1)) == 'x')
{
text = JSTokenizer._hexadecimalChars;
base.MoveAhead(2);
}
}
bool flag = false;
bool flag2 = false;
while (TextParser.StringArrayContains(text, base.Peek()))
{
if (base.Peek() == '.')
{
if (flag)
{
break;
}
flag = true;
}
else if (char.ToLower(base.Peek()) == 'e' && text == JSTokenizer._decimalChars)
{
if (flag2)
{
break;
}
flag2 = true;
if (base.Peek(1) == '+' || base.Peek(1) == '-')
{
base.MoveAhead();
}
}
base.MoveAhead();
}
}
else if (base.Peek() == '/' && jSToken.PreviousType != JSTokenTypes.CloseParen && jSToken.PreviousType != JSTokenTypes.CloseBracket && (jSToken.PreviousType != JSTokenTypes.Symbol || jSToken.PreviousValue == "return" || jSToken.PreviousValue == "yield") && jSToken.PreviousType != JSTokenTypes.String && jSToken.PreviousType != JSTokenTypes.Number && jSToken.PreviousType != JSTokenTypes.RegEx && jSToken.PreviousType != JSTokenTypes.Dot && jSToken.PreviousType != JSTokenTypes.UnarySuffix)
{
base.MoveAhead();
jSToken.Type = JSTokenTypes.RegEx;
bool flag3 = false;
while (!base.EndOfText && (base.Peek() != '/' || flag3) && base.Peek() != '\r' && base.Peek() != '\n')
{
if (base.Peek() == '[')
{
flag3 = true;
}
else if (base.Peek() == ']')
{
flag3 = false;
}
else if (base.Peek() == '\\')
{
char c2 = base.Peek(1);
if (c2 == '\\' || c2 == '/' || c2 == ']' || c2 == '[')
{
base.MoveAhead();
}
}
base.MoveAhead();
}
if (base.Peek() == '/')
{
base.MoveAhead();
while (char.IsLetter(base.Peek()))
{
base.MoveAhead();
}
}
}
else if (TextParser.StringArrayContains(JSTokenizer._operatorChars, base.Peek()))
{
char c3 = base.Peek();
bool flag4 = false;
JSTokenizer.NamedOperator[] named = JSTokenizer._named;
for (int i = 0; i < named.Length; i++)
{
JSTokenizer.NamedOperator namedOperator = named[i];
if (namedOperator.Operator == c3)
{
jSToken.Type = namedOperator.Type;
base.MoveAhead();
flag4 = true;
break;
}
}
if (!flag4)
{
string text2 = null;
string[] multiCharOperators = JSTokenizer._multiCharOperators;
for (int j = 0; j < multiCharOperators.Length; j++)
{
string text3 = multiCharOperators[j];
if (base.MatchesCurrent(text3))
{
text2 = text3;
break;
}
}
if (text2 != null)
{
base.MoveAhead(text2.Length);
}
else
{
text2 = base.Peek().ToString();
base.MoveAhead();
}
if (text2 == "~" || text2 == "!")
{
jSToken.Type = JSTokenTypes.UnaryPrefix;
}
else if (text2 == "++" || text2 == "--")
{
if (jSToken.PreviousType == JSTokenTypes.Symbol || jSToken.PreviousType == JSTokenTypes.CloseParen || jSToken.PreviousType == JSTokenTypes.CloseBracket)
{
jSToken.Type = JSTokenTypes.UnarySuffix;
}
else
{
jSToken.Type = JSTokenTypes.UnaryPrefix;
}
}
else if (text2 == "+" || text2 == "-")
{
if (jSToken.PreviousType == JSTokenTypes.Symbol || jSToken.PreviousType == JSTokenTypes.Number || jSToken.PreviousType == JSTokenTypes.String || jSToken.PreviousType == JSTokenTypes.RegEx || jSToken.PreviousType == JSTokenTypes.CloseParen || jSToken.PreviousType == JSTokenTypes.CloseBracket)
{
jSToken.Type = JSTokenTypes.BinaryOperator;
}
else
{
jSToken.Type = JSTokenTypes.UnaryPrefix;
}
}
else
{
jSToken.Type = JSTokenTypes.BinaryOperator;
}
}
}
else
{
jSToken.Type = JSTokenTypes.Unknown;
base.MoveAhead();
}
jSToken.Value = base.Extract(position, base.Position);
this.Token = jSToken;
return(true);
}
