コード例 #1
0
        public void InitTokenizer(string s)
        {
            _inputString = s;
            _position    = 0;
            _tokenType   = SoqlTokenType.BOF;

            GetNextToken();
        }
コード例 #2
0
        public void Expect(SoqlTokenType type)
        {
            if (_tokenType != type)
            {
                throw new SoqlException("Expected token of type: " + type + ", got " + _tokenType + " (" + _tokenValue + ").", TokenPosition);
            }

            GetNextToken();
        }
コード例 #3
0
 public CharToTokenType(char ch, SoqlTokenType tokenType)
 {
     this.ch = ch;
     this.tokenType = tokenType;
 }
コード例 #4
0
 public bool IsToken(SoqlTokenType token)
 {
     return _tokenType == token;
 }
コード例 #5
0
        public void Expect(SoqlTokenType type)
        {
            if (_tokenType != type)
                throw new SoqlException("Expected token of type: " + type + ", got " + _tokenType + " (" + _tokenValue + ").", TokenPosition);

            GetNextToken();
        }
コード例 #6
0
        public void InitTokenizer(string s)
        {
            _inputString = s;
            _position = 0;
            _tokenType = SoqlTokenType.BOF;

            GetNextToken();
        }
コード例 #7
0
        public void GetNextToken()
        {
            if (_tokenType == SoqlTokenType.EOF)
            {
                throw new Exception("Cannot read past end of stream.");
            }

            if (IgnoreWhiteSpace)
            {
                SkipWhitespace();
            }
            ;

            _tokenPosition = _position;

            int i = PeekChar();

            if (i == -1)
            {
                TokenType = SoqlTokenType.EOF;
                return;
            }

            char ch = (char)i;

            if (!IgnoreWhiteSpace && Char.IsWhiteSpace(ch))
            {
                StringBuilder sb = new StringBuilder();
                int           ch2;

                while ((ch2 = PeekChar()) != -1)
                {
                    if (!Char.IsWhiteSpace((char)ch2))
                    {
                        break;
                    }

                    sb.Append((char)ch2);
                    ReadChar();
                }
                ;

                TokenType   = SoqlTokenType.Whitespace;
                _tokenValue = sb.ToString();
                return;
            }

            if (Char.IsDigit(ch))
            {
                TokenType = SoqlTokenType.Number;
                string s = "";

                s += ch;
                ReadChar();

                while ((i = PeekChar()) != -1)
                {
                    ch = (char)i;

                    if (Char.IsDigit(ch) || (ch == '.'))
                    {
                        s += (char)ReadChar();
                    }
                    else
                    {
                        break;
                    };
                }
                ;

                _tokenValue = s;
                return;
            }

            if (ch == '\'')
            {
                TokenType = SoqlTokenType.String;

                string s = "";

                s += ch;
                ReadChar();

                while ((i = PeekChar()) != -1)
                {
                    ch = (char)i;

                    s += (char)ReadChar();

                    if (ch == '\'')
                    {
                        if (PeekChar() == (int)'\'')
                        {
                            s += '\'';
                            ReadChar();
                        }
                        else
                        {
                            break;
                        }
                    }
                }
                ;

                _tokenValue = s;
                return;
            }

            if (ch == '_' || Char.IsLetter(ch))
            {
                TokenType = SoqlTokenType.Keyword;

                StringBuilder sb = new StringBuilder();

                sb.Append((char)ch);

                ReadChar();

                while ((i = PeekChar()) != -1)
                {
                    if ((char)i == '_' || Char.IsLetterOrDigit((char)i))
                    {
                        sb.Append((char)ReadChar());
                    }
                    else
                    {
                        break;
                    };
                }
                ;

                _tokenValue          = sb.ToString();
                _tokenValueLowercase = _tokenValue.ToLower();
                return;
            }

            ReadChar();
            _tokenValue = ch.ToString();

            if (ch == '<' && PeekChar() == (int)'>')
            {
                TokenType   = SoqlTokenType.NE;
                _tokenValue = "<>";
                ReadChar();
                return;
            }

            if (ch == '!' && PeekChar() == (int)'=')
            {
                TokenType   = SoqlTokenType.NE;
                _tokenValue = "!=";
                ReadChar();
                return;
            }

            if (ch == '&' && PeekChar() == (int)'&')
            {
                TokenType   = SoqlTokenType.And;
                _tokenValue = "&&";
                ReadChar();
                return;
            }

            if (ch == '|' && PeekChar() == (int)'|')
            {
                TokenType   = SoqlTokenType.Or;
                _tokenValue = "||";
                ReadChar();
                return;
            }

            if (ch == '<' && PeekChar() == (int)'=')
            {
                TokenType   = SoqlTokenType.LE;
                _tokenValue = "<=";
                ReadChar();
                return;
            }

            if (ch == '>' && PeekChar() == (int)'=')
            {
                TokenType   = SoqlTokenType.GE;
                _tokenValue = ">=";
                ReadChar();
                return;
            }

            if (ch == '=' && PeekChar() == (int)'=')
            {
                TokenType   = SoqlTokenType.EQ;
                _tokenValue = "==";
                ReadChar();
                return;
            }

            if (ch >= 32 && ch < 128)
            {
                SoqlTokenType tt = charIndexToTokenType[ch];

                if (tt != SoqlTokenType.Invalid)
                {
                    TokenType   = tt;
                    _tokenValue = new String(ch, 1);
                    return;
                }
                else
                {
                    throw new Exception("Invalid punctuation: " + ch);
                }
            }
            throw new Exception("Invalid token: " + ch);
        }
コード例 #8
0
 public CharToTokenType(char ch, SoqlTokenType tokenType)
 {
     this.ch        = ch;
     this.tokenType = tokenType;
 }
コード例 #9
0
 public bool IsToken(SoqlTokenType token)
 {
     return(_tokenType == token);
 }