public bool GetToken() { if (this.PendingToken != null) { this.Token = this.PendingToken; this.PendingToken = null; return(true); } base.MovePastInlineWhitespace(); if (base.EndOfText) { return(false); } CSSToken cSSToken = new CSSToken(); if (this.Token != null) { cSSToken.PreviousValue = this.Token.Value; cSSToken.PreviousType = this.Token.Type; } int position = base.Position; if (base.Peek() == '\r' || base.Peek() == '\n') { cSSToken.Type = CSSTokenTypes.CRLF; base.MoveAhead(); if (base.Peek() == '\r' || base.Peek() == '\n') { base.MoveAhead(); } base.MovePastInlineWhitespace(); } else if (TextParser.StringArrayContains(CSSTokenizer._symbolChars, base.Peek())) { cSSToken.Type = CSSTokenTypes.Symbol; base.MoveAhead(); while (TextParser.StringArrayContains(CSSTokenizer._symbolChars, base.Peek())) { base.MoveAhead(); } } else if (base.Peek() == '/' && base.Peek(1) == '*') { cSSToken.Type = CSSTokenTypes.InlineComment; base.MoveTo("*/"); base.MoveAhead(2); } else if (base.Peek() == '\'' || base.Peek() == '"' || base.Peek() == '(' || base.Peek() == ')') { cSSToken.Type = CSSTokenTypes.String; char c = base.Peek(); if (c == '(') { c = ')'; } base.MoveAhead(); while (!base.EndOfText) { if (base.Peek() == c) { base.MoveAhead(); break; } if (base.Peek() == '\\' && (base.Peek(1) == '\\' || base.Peek(1) == c)) { base.MoveAhead(2); } else { base.MoveAhead(); } } } else if (char.IsDigit(base.Peek()) || (base.Peek() == '.' && char.IsDigit(base.Peek(1)))) { cSSToken.Type = CSSTokenTypes.Number; string decimalChars = CSSTokenizer._decimalChars; bool flag = false; while (TextParser.StringArrayContains(decimalChars, base.Peek())) { if (base.Peek() == '.') { if (flag) { break; } flag = true; } base.MoveAhead(); } } else if (TextParser.StringArrayContains(CSSTokenizer._operatorChars, base.Peek())) { char c2 = base.Peek(); bool flag2 = false; CSSTokenizer.NamedOperator[] named = CSSTokenizer._named; for (int i = 0; i < named.Length; i++) { CSSTokenizer.NamedOperator namedOperator = named[i]; if (namedOperator.Operator == c2) { cSSToken.Type = namedOperator.Type; base.MoveAhead(); flag2 = true; break; } } if (!flag2) { if (c2 == '+' || c2 == '-') { cSSToken.Type = CSSTokenTypes.UnaryPrefix; } else { cSSToken.Type = CSSTokenTypes.Unknown; } base.MoveAhead(); } } else { cSSToken.Type = CSSTokenTypes.Unknown; base.MoveAhead(); } cSSToken.Value = base.Extract(position, base.Position); this.Token = cSSToken; return(true); }
public bool GetToken() { if (this.PendingToken != null) { this.Token = this.PendingToken; this.PendingToken = null; return(true); } base.MovePastInlineWhitespace(); if (base.EndOfText) { return(false); } JSToken jSToken = new JSToken(); if (this.Token != null) { jSToken.PreviousValue = this.Token.Value; jSToken.PreviousType = this.Token.Type; } int position = base.Position; if (base.Peek() == '\r' || base.Peek() == '\n') { jSToken.Type = JSTokenTypes.CRLF; base.MoveAhead(); if (base.Peek() == '\r' || base.Peek() == '\n') { base.MoveAhead(); } base.MovePastInlineWhitespace(); } else if (TextParser.StringArrayContains(JSTokenizer._firstSymbolChars, base.Peek())) { jSToken.Type = JSTokenTypes.Symbol; base.MoveAhead(); while (TextParser.StringArrayContains(JSTokenizer._symbolChars, base.Peek())) { base.MoveAhead(); } } else if (base.Peek() == '/' && base.Peek(1) == '/') { jSToken.Type = JSTokenTypes.FullLineComment; base.MoveToEndOfLine(); } else if (base.Peek() == '<' && base.Peek(1) == '!' && base.Peek(2) == '-' && base.Peek(3) == '-') { jSToken.Type = JSTokenTypes.FullLineComment; base.MoveToEndOfLine(); } else if (base.Peek() == '/' && base.Peek(1) == '*') { jSToken.Type = JSTokenTypes.InlineComment; base.MoveTo("*/"); base.MoveAhead(2); } else if (base.Peek() == '\'' || base.Peek() == '"') { jSToken.Type = JSTokenTypes.String; char c = base.Peek(); base.MoveAhead(); while (!base.EndOfText) { if (base.Peek() == c) { base.MoveAhead(); break; } if (base.Peek() == '\\' && (base.Peek(1) == '\\' || base.Peek(1) == c)) { base.MoveAhead(2); } else { base.MoveAhead(); } } } else if (char.IsDigit(base.Peek()) || (base.Peek() == '.' && char.IsDigit(base.Peek(1)))) { jSToken.Type = JSTokenTypes.Number; string text = JSTokenizer._decimalChars; if (base.Peek() == '0') { if (char.IsDigit(base.Peek(1))) { text = JSTokenizer._octalChars; } else if (char.ToLower(base.Peek(1)) == 'x') { text = JSTokenizer._hexadecimalChars; base.MoveAhead(2); } } bool flag = false; bool flag2 = false; while (TextParser.StringArrayContains(text, base.Peek())) { if (base.Peek() == '.') { if (flag) { break; } flag = true; } else if (char.ToLower(base.Peek()) == 'e' && text == JSTokenizer._decimalChars) { if (flag2) { break; } flag2 = true; if (base.Peek(1) == '+' || base.Peek(1) == '-') { base.MoveAhead(); } } base.MoveAhead(); } } else if (base.Peek() == '/' && jSToken.PreviousType != JSTokenTypes.CloseParen && jSToken.PreviousType != JSTokenTypes.CloseBracket && (jSToken.PreviousType != JSTokenTypes.Symbol || jSToken.PreviousValue == "return" || jSToken.PreviousValue == "yield") && jSToken.PreviousType != JSTokenTypes.String && jSToken.PreviousType != JSTokenTypes.Number && jSToken.PreviousType != JSTokenTypes.RegEx && jSToken.PreviousType != JSTokenTypes.Dot && jSToken.PreviousType != JSTokenTypes.UnarySuffix) { base.MoveAhead(); jSToken.Type = JSTokenTypes.RegEx; bool flag3 = false; while (!base.EndOfText && (base.Peek() != '/' || flag3) && base.Peek() != '\r' && base.Peek() != '\n') { if (base.Peek() == '[') { flag3 = true; } else if (base.Peek() == ']') { flag3 = false; } else if (base.Peek() == '\\') { char c2 = base.Peek(1); if (c2 == '\\' || c2 == '/' || c2 == ']' || c2 == '[') { base.MoveAhead(); } } base.MoveAhead(); } if (base.Peek() == '/') { base.MoveAhead(); while (char.IsLetter(base.Peek())) { base.MoveAhead(); } } } else if (TextParser.StringArrayContains(JSTokenizer._operatorChars, base.Peek())) { char c3 = base.Peek(); bool flag4 = false; JSTokenizer.NamedOperator[] named = JSTokenizer._named; for (int i = 0; i < named.Length; i++) { JSTokenizer.NamedOperator namedOperator = named[i]; if (namedOperator.Operator == c3) { jSToken.Type = namedOperator.Type; base.MoveAhead(); flag4 = true; break; } } if (!flag4) { string text2 = null; string[] multiCharOperators = JSTokenizer._multiCharOperators; for (int j = 0; j < multiCharOperators.Length; j++) { string text3 = multiCharOperators[j]; if (base.MatchesCurrent(text3)) { text2 = text3; break; } } if (text2 != null) { base.MoveAhead(text2.Length); } else { text2 = base.Peek().ToString(); base.MoveAhead(); } if (text2 == "~" || text2 == "!") { jSToken.Type = JSTokenTypes.UnaryPrefix; } else if (text2 == "++" || text2 == "--") { if (jSToken.PreviousType == JSTokenTypes.Symbol || jSToken.PreviousType == JSTokenTypes.CloseParen || jSToken.PreviousType == JSTokenTypes.CloseBracket) { jSToken.Type = JSTokenTypes.UnarySuffix; } else { jSToken.Type = JSTokenTypes.UnaryPrefix; } } else if (text2 == "+" || text2 == "-") { if (jSToken.PreviousType == JSTokenTypes.Symbol || jSToken.PreviousType == JSTokenTypes.Number || jSToken.PreviousType == JSTokenTypes.String || jSToken.PreviousType == JSTokenTypes.RegEx || jSToken.PreviousType == JSTokenTypes.CloseParen || jSToken.PreviousType == JSTokenTypes.CloseBracket) { jSToken.Type = JSTokenTypes.BinaryOperator; } else { jSToken.Type = JSTokenTypes.UnaryPrefix; } } else { jSToken.Type = JSTokenTypes.BinaryOperator; } } } else { jSToken.Type = JSTokenTypes.Unknown; base.MoveAhead(); } jSToken.Value = base.Extract(position, base.Position); this.Token = jSToken; return(true); }