Пример #1
0
 public ChoExpressionTokenizerState(ChoTokenType currentToken, string tokenText, ChoPosition currentPosition)
 {
     _currentToken    = currentToken;
     _tokenText       = tokenText;
     _currentPosition = currentPosition;
 }
 internal ChoExpressionTokenizerState(eSquare.Core.ExpresssionEvaluator.ChoExpressionTokenizer.ChoTokenType currentToken, string tokenText, ChoPosition currentPosition)
 {
     _currentToken    = currentToken;
     _tokenText       = tokenText;
     _currentPosition = currentPosition;
 }
Пример #3
0
        public void GetNextToken()
        {
            if (_tokenType == ChoTokenType.EOF)
            {
                throw new ChoExpressionParseException("Cannot read past end of stream.");
            }

            if (IgnoreWhitespace)
            {
                SkipWhitespace();
            }

            _tokenStartPosition = new ChoPosition(_position);

            int nextChar = PeekChar();

            if (nextChar == -1)
            {
                _tokenType = ChoTokenType.EOF;
                return;
            }

            char ch = (char)nextChar;

            if (!SingleCharacterMode)
            {
                #region Handle WhiteSpace chars

                if (!IgnoreWhitespace && Char.IsWhiteSpace(ch))
                {
                    StringBuilder sb = new StringBuilder();
                    int           ch2;

                    while ((ch2 = PeekChar()) != -1)
                    {
                        if (!Char.IsWhiteSpace((char)ch2))
                        {
                            break;
                        }

                        sb.Append((char)ch2);
                        ReadChar();
                    }
                    ;

                    _tokenType = ChoTokenType.Whitespace;
                    _tokenText = sb.ToString();
                    return;
                }

                #endregion Handle WhiteSpace chars

                #region Handle Digits

                if (Char.IsDigit(ch))
                {
                    _tokenType = ChoTokenType.Number;

                    StringBuilder sb = new StringBuilder();
                    //sb.Append(ch);

                    while ((nextChar = PeekChar()) != -1)
                    {
                        ch = (char)nextChar;

                        if (!Char.IsDigit(ch))
                        {
                            break;
                        }

                        sb.Append(ch);
                        ReadChar();
                    }
                    ;

                    _tokenText = sb.ToString();
                    return;
                }

                #endregion Handle Digits

                #region Handle String in quotes

                if (ch == '\'')
                {
                    _tokenType = ChoTokenType.String;

                    StringBuilder sb = new StringBuilder();
                    ReadChar();
                    while ((nextChar = ReadChar()) != -1)
                    {
                        ch = (char)nextChar;

                        if (ch == '\'')
                        {
                            if (PeekChar() == (int)'\'')
                            {
                                ReadChar();
                            }
                            else
                            {
                                break;
                            }
                        }
                        sb.Append(ch);
                    }
                    ;

                    _tokenText = sb.ToString();
                    return;
                }

                #endregion Handle String in quotes

                #region Handle Keywords

                if (ch == '_' || Char.IsLetter(ch))
                {
                    _tokenType = ChoTokenType.Keyword;

                    StringBuilder sb = new StringBuilder();

                    sb.Append((char)ch);

                    ReadChar();

                    while ((nextChar = PeekChar()) != -1)
                    {
                        if ((char)nextChar == '_' || (char)nextChar == '-' || Char.IsLetterOrDigit((char)nextChar))
                        {
                            sb.Append((char)ReadChar());
                        }
                        else
                        {
                            break;
                        }
                    }
                    ;

                    _tokenText = sb.ToString();
                    if (_tokenText.EndsWith("-"))
                    {
                        throw new ChoExpressionParseException(String.Format(CultureInfo.InvariantCulture,
                                                                            "Identifier cannot end with a dash: {0}", _tokenText), CurrentPosition.CharIndex);
                    }
                    return;
                }

                #endregion Handle Keywords

                ReadChar();

                if (ch == ':' && PeekChar() == (int)':')
                {
                    _tokenType = ChoTokenType.DoubleColon;
                    _tokenText = "::";
                    ReadChar();
                    return;
                }

                if (ch == '!' && PeekChar() == (int)'=')
                {
                    _tokenType = ChoTokenType.NE;
                    _tokenText = "!=";
                    ReadChar();
                    return;
                }

                if (ch == '=' && PeekChar() == (int)'=')
                {
                    _tokenType = ChoTokenType.EQ;
                    _tokenText = "==";
                    ReadChar();
                    return;
                }

                if (ch == '<' && PeekChar() == (int)'=')
                {
                    _tokenType = ChoTokenType.LE;
                    _tokenText = "<=";
                    ReadChar();
                    return;
                }

                if (ch == '>' && PeekChar() == (int)'=')
                {
                    _tokenType = ChoTokenType.GE;
                    _tokenText = ">=";
                    ReadChar();
                    return;
                }
            }
            else
            {
                ReadChar();
            }
            _tokenText = new String(ch, 1);
            _tokenType = ChoTokenType.Punctuation;
            if (ch >= 32 && ch < 128)
            {
                _tokenType = charIndexToTokenType[ch];
            }
        }