/// <summary>
        /// Reads the next token from the character
        /// sequence being tokenized.
        /// </summary>
        private void ReadToken()
        {
            if (m_wait)
            {
                m_wait = false;
                m_currentIndex = m_nextTokenIndex;

                return;
            }

            if (!m_inIdentifierChain)
            {
                m_identifierChain = null;
                m_identifierChainPredecessor = null;
                m_identifierChainPredecessorType = SqlTokenType.None;
            }

            while (m_currentIndex < m_charsLength
                   && JavaCharacter.isWhitespace(m_chars[m_currentIndex]))
            {
                m_currentIndex++;
            }

            m_token = string.Empty;
            m_parameterName = string.Empty;
            m_parameterNamePrefix = ' ';
            m_tokenIndex = m_currentIndex;

            if (m_currentIndex >= m_charsLength)
            {
                // character sequence has been exhausted.
                m_tokenType = SqlTokenType.None;

                return;
            }

            char c = m_chars[m_currentIndex];
            bool point = false;
            bool digit = false;
            bool exp = false;
            bool afterexp = false;
            bool end = false;
            char cfirst = (char)0;

            m_wasLastTokenDelimited = false;

            if (JavaCharacter.isJavaIdentifierStart(c))
            {
                m_tokenType = SqlTokenType.Name;
            }
            else if (JavaCharacter.isDigit(c))
            {
                m_tokenType = SqlTokenType.NumberLiteral;
                digit = true;
            }
            else
            {
                switch (c)
                {
                    case '(':
                        {
                            m_token = Token.ValueFor.OPENBRACKET;
                            m_tokenType = SqlTokenType.Special;

                            m_currentIndex++;

                            return;
                        }
                    case ')':
                        {
                            m_token = Token.ValueFor.CLOSEBRACKET;
                            m_tokenType = SqlTokenType.Special;

                            m_currentIndex++;

                            return;
                        }
                    case ',':
                        {
                            m_token = Token.ValueFor.COMMA;
                            m_tokenType = SqlTokenType.Special;

                            m_currentIndex++;

                            return;
                        }
                    case '*':
                        {
                            m_token = Token.ValueFor.MULTIPLY;
                            m_tokenType = SqlTokenType.Special;

                            m_currentIndex++;

                            return;
                        }
                    case '=':
                        {
                            m_token = Token.ValueFor.EQUALS;
                            m_tokenType = SqlTokenType.Special;

                            m_currentIndex++;

                            return;
                        }
                    case ';':
                        {
                            m_token = Token.ValueFor.SEMICOLON;
                            m_tokenType = SqlTokenType.Special;

                            m_currentIndex++;

                            return;
                        }
                    case '+':
                        {
                            m_token = Token.ValueFor.PLUS;
                            m_tokenType = SqlTokenType.Special;

                            m_currentIndex++;

                            return;
                        }
                    case '%':
                        {
                            m_token = Token.ValueFor.PERCENT;
                            m_tokenType = SqlTokenType.Special;

                            m_currentIndex++;

                            return;
                        }
                    case '?':
                        {
                            if (!m_acceptParameterMarkers)
                            {
                                throw Tokenizer.UnexpectedToken(c);
                            }

                            m_token = Token.ValueFor.QUESTION;
                            m_tokenType = SqlTokenType.Special;

                            m_currentIndex++;

                            return;
                        }
                    case ':':
                        {
                            if (!m_acceptNamedParameters)
                            {
                                throw UnexpectedToken(c);
                            }

                            if (!(++m_currentIndex < m_charsLength))
                            {
                                throw Tokenizer.UnexpectedEndOfCommand();
                            }

                            c = m_chars[m_currentIndex];

                            if (!JavaCharacter.isJavaIdentifierStart(c))
                            {
                                throw Tokenizer.InvalidIdentifier(":" + c);
                            }

                            m_tokenType = SqlTokenType.NamedParameter;
                            m_parameterNamePrefix = ':';

                            break;
                        }
                    case '@':
                        {
                            if (!m_acceptNamedParameters)
                            {
                                throw UnexpectedToken(c);
                            }

                            if (!(++m_currentIndex < m_charsLength))
                            {
                                throw Tokenizer.UnexpectedEndOfCommand();
                            }

                            c = m_chars[m_currentIndex];

                            if (!JavaCharacter.isJavaIdentifierStart(c))
                            {
                                throw InvalidIdentifier(":" + c);
                            }

                            m_tokenType = SqlTokenType.NamedParameter;
                            m_parameterNamePrefix = '@';

                            break;
                        }
                    case '"':
                        {
                            m_wasLastTokenDelimited = true;
                            m_tokenType = SqlTokenType.DelimitedIdentifier;

                            m_currentIndex++;

                            m_token = GetAsQuotedString('"');

                            if (m_currentIndex == m_chars.Length)
                            {
                                if (m_inIdentifierChain)
                                {
                                    m_identifierChain.Add(new Token(m_token, m_tokenType));
                                    m_inIdentifierChain = false;
                                    m_tokenType = SqlTokenType.IdentifierChain;
                                }

                                return;
                            }

                            c = m_chars[m_currentIndex];

                            if (c == '.')
                            {
                                ReadIdentifierChain();
                            }
                            else if (m_inIdentifierChain)
                            {
                                m_identifierChain.Add(new Token(m_token, m_tokenType));
                                m_inIdentifierChain = false;
                                m_tokenType = SqlTokenType.IdentifierChain;
                            }

                            return;
                        }
                    case '\'':
                        {
                            m_tokenType = SqlTokenType.StringLiteral;

                            m_currentIndex++;

                            m_token = GetAsQuotedString('\'');

                            return;
                        }
                    case '!':
                    case '<':
                    case '>':
                    case '|':
                    case '/':
                    case '-':
                        {
                            cfirst = c;
                            m_tokenType = SqlTokenType.Special;

                            break;
                        }
                    case '.':
                        {
                            m_tokenType = SqlTokenType.DecimalLiteral;
                            point = true;

                            break;
                        }
                    default:
                        {
                            throw UnexpectedToken(c);
                        }
                }
            }

            int start = m_currentIndex++;

            while (true)
            {
                if (m_currentIndex >= m_charsLength)
                {
                    c = ' ';
                    end = true;

                    switch (m_tokenType)
                    {
                        case SqlTokenType.StringLiteral:
                        case SqlTokenType.DelimitedIdentifier:
                            {
                                throw Tokenizer.UnexpectedEndOfCommand();
                            }
                    }
                }
                else
                {
                    c = m_chars[m_currentIndex];
                }

                switch (m_tokenType)
                {
                    case SqlTokenType.NamedParameter:
                    case SqlTokenType.Name:
                        {
                            if (JavaCharacter.isJavaIdentifierPart(c))
                            {
                                break;
                            }

                            m_token = m_chars.Substring(start, m_currentIndex
                                - start).ToUpper(m_EnglishCulture);

                            if (m_tokenType == SqlTokenType.NamedParameter)
                            {
                                m_parameterName = m_chars.Substring(start,
                                        (m_currentIndex - start));

                                return;
                            }

                            // only for Name, not for NamedParameter
                            if (c == '.')
                            {
                                ReadIdentifierChain();
                            }
                            else if (m_inIdentifierChain /*&& c!= '.'*/)
                            {
                                m_identifierChain.Add(new Token(m_token, m_tokenType));
                                m_inIdentifierChain = false;
                                m_tokenType = SqlTokenType.IdentifierChain;
                            }
                            else if (c == '(')
                            {
                                // no-op
                                // potentially a function call, SQL operator, etc.
                            }
                            else
                            {
                                // if in value list then it is a value
                                int type = m_ValueTokens.get(m_token, -1);

                                if (type != -1)
                                {
                                    m_tokenType = (SqlTokenType)type;
                                }
                            }

                            return;
                        }
                    case SqlTokenType.DelimitedIdentifier:
                    case SqlTokenType.StringLiteral:
                        {
                            // should never get here
                            break;
                        }
                    case SqlTokenType.Remark:
                        {
                            if (end)
                            {
                                // unfinished remark
                                // TODO: log/trace error condition here
                                m_tokenType = SqlTokenType.None;

                                return;
                            }
                            else if (c == '*')
                            {
                                m_currentIndex++;

                                if (m_currentIndex < m_charsLength
                                    && m_chars[m_currentIndex] == '/')
                                {
                                    // TODO: eliminate recursion
                                    m_currentIndex++;

                                    ReadToken();

                                    return;
                                }
                            }

                            break;
                        }
                    case SqlTokenType.RemarkLine:
                        {
                            if (end)
                            {
                                m_tokenType = SqlTokenType.None;

                                return;
                            }
                            else if (c == '\r' || c == '\n')
                            {
                                // TODO: eliminate recursion
                                ReadToken();

                                return;
                            }

                            break;
                        }
                    case SqlTokenType.Special:
                        {
                            if (c == '/' && cfirst == '/')
                            {
                                m_tokenType = SqlTokenType.RemarkLine;

                                break;
                            }
                            else if (c == '-' && cfirst == '-')
                            {
                                m_tokenType = SqlTokenType.RemarkLine;

                                break;
                            }
                            else if (c == '*' && cfirst == '/')
                            {
                                m_tokenType = SqlTokenType.Remark;

                                break;
                            }
                            else if (c == '>' || c == '=' || c == '|')
                            {
                                break;
                            }

                            m_token
                                = m_chars.Substring(start,
                                                    m_currentIndex - start);

                            return;
                        }
                    case SqlTokenType.NumberLiteral:
                    case SqlTokenType.FloatLiteral:
                    case SqlTokenType.DecimalLiteral:
                        {
                            if (JavaCharacter.isDigit(c))
                            {
                                digit = true;
                            }
                            else if (c == '.')
                            {
                                if (point)
                                {
                                    throw Tokenizer.UnexpectedToken(c);
                                }

                                m_tokenType = SqlTokenType.DecimalLiteral;

                                point = true;
                            }
                            else if (c == 'E' || c == 'e')
                            {
                                if (exp)
                                {
                                    throw UnexpectedToken(c);
                                }

                                m_tokenType = SqlTokenType.FloatLiteral;

                                // first character after exp may be + or -
                                afterexp = true;
                                point = true;
                                exp = true;
                            }
                            else if (c == '-' && afterexp)
                            {
                                afterexp = false;
                            }
                            else if (c == '+' && afterexp)
                            {
                                afterexp = false;
                            }
                            else
                            {
                                //afterexp = false;

                                if (!digit)
                                {
                                    if (point
                                        && (start == (m_currentIndex - 1)))
                                    {
                                        m_token = Token.ValueFor.PERIOD;
                                        m_tokenType = SqlTokenType.Special;

                                        return;
                                    }

                                    throw UnexpectedToken(c);
                                }

                                m_token = m_chars.Substring(start,
                                    (m_currentIndex - start));

                                return;
                            }

                            break;
                        }
                }

                m_currentIndex++;
            }
        }
        /// <summary>
        /// Resets this tokenizer with the given character sequence.
        /// </summary>
        /// <param name="chars">
        /// The new character sequence to be tokenized.
        /// </param>
        /// <exception cref="ArgumentNullException">
        /// When <c>chars</c> is <c>null</c>.
        /// </exception>
        public void Reset(string chars)
        {
            if (chars == null)
            {
                throw new ArgumentNullException("chars");
            }

            m_chars = chars;
            m_charsLength = chars.Length;
            m_currentIndex = 0;
            m_tokenIndex = 0;
            m_nextTokenIndex = 0;
            m_beginIndex = 0;
            m_tokenType = SqlTokenType.None;
            m_identifierChainPredecessorType = SqlTokenType.None;
            m_token = null;
            m_parameterName = string.Empty;
            m_parameterNamePrefix = ' ';
            m_identifierChainPredecessor = null;
            m_wait = false;
            m_wasLastTokenDelimited = false;
            m_inIdentifierChain = false;
        }
        /// <summary>
        /// Reads an entire identifier chain.
        /// </summary>
        private void ReadIdentifierChain()
        {
            if (m_identifierChain == null)
            {
                m_identifierChain = new GenericTokenList();
            }

            if (m_inIdentifierChain && m_enforceTwoPartIdentifierChain)
            {
                throw Tokenizer.IdentiferChainLengthExceeded();
            }

            m_identifierChainPredecessor = m_token;
            m_identifierChainPredecessorType = m_tokenType;
            m_inIdentifierChain = true;

            m_currentIndex++;

            m_identifierChain.Add(new Token(m_token, m_tokenType));

            // TODO: avoid recursion
            // Also, this has problems when there is whitespace
            // after the dot; same with NAME
            ReadToken();
        }
        /// <summary>
        /// Retrieves the SQL literal value of the given token value.
        /// </summary>
        /// <remarks>
        /// Note that, as a ref parameter, the tokenType may be
        /// re-assigned as part of computing the literal value.
        /// </remarks>
        /// <param name="tokenType">
        /// Type of the token.
        /// </param>
        /// <param name="tokenValue">
        /// The token value, as a character sequence
        /// </param>
        /// <returns>
        /// the token literal value, as an object representing the SQL literal
        /// numeric, temporal, boolean or character value.
        /// </returns>
        public static object ToLiteralValue(ref SqlTokenType tokenType,
            string tokenValue)
        {
            switch (tokenType)
            {
                case SqlTokenType.Null:
                    {
                        return null;
                    }
                case SqlTokenType.StringLiteral:
                    {
                        if (tokenValue.Length <= 10 &&
                            Tokenizer.DateLiteralRegex.IsMatch(tokenValue))
                        {
                            try
                            {
                                object rval = HsqlDateTime.dateValue(tokenValue);

                                tokenType = SqlTokenType.DateLiteral;

                                return rval;
                            }
                            catch (Exception)
                            {
                            }
                        }
                        else if (tokenValue.Length <= 10 &&
                            Tokenizer.TimeLiteralRegex.IsMatch(tokenValue))
                        {
                            try
                            {
                                object rval = HsqlDateTime.timeValue(tokenValue);

                                tokenType = SqlTokenType.TimeLiteral;

                                return rval;
                            }
                            catch (Exception)
                            {
                            }
                        }
                        else if (Tokenizer.TimestampLiteralRegex.IsMatch(tokenValue))
                        {
                            try
                            {
                                object rval = HsqlDateTime.timestampValue(tokenValue);

                                tokenType = SqlTokenType.TimestampLiteral;

                                return rval;
                            }
                            catch (Exception)
                            {
                            }
                        }

                        return tokenValue;
                    }
                case SqlTokenType.BigIntLiteral:
                    {
                        return HsqlValuePool.getLong(JavaLong.parseLong(tokenValue));
                    }
                case SqlTokenType.NumberLiteral:
                    {
                        // Returns unsigned values which may later need to be negated.
                        // As a result, Integer.MIN_VALUE or Long.MIN_VALUE are promoted
                        // to a wider type.
                        if (tokenValue.Length < 11)
                        {
                            try
                            {
                                object rval = HsqlValuePool.getInt(
                                    JavaInteger.parseInt(tokenValue));

                                tokenType = SqlTokenType.IntegerLiteral;

                                return rval;
                            }
                            catch (Exception)
                            {
                            }
                        }

                        if (tokenValue.Length < 20)
                        {
                            try
                            {
                                object rval = HsqlValuePool.getLong(
                                    JavaLong.parseLong(tokenValue));

                                tokenType = SqlTokenType.BigIntLiteral;

                                return rval;
                            }
                            catch (Exception) { }
                        }

                        tokenType = SqlTokenType.DecimalLiteral;

                        return new JavaBigDecimal(tokenValue);
                    }
                case SqlTokenType.FloatLiteral:
                    {
                        double d = JavaSystem.parseDouble(tokenValue);
                        long l = JavaDouble.doubleToLongBits(d);

                        return HsqlValuePool.getDouble(l);
                    }
                case SqlTokenType.DecimalLiteral:
                    {
                        return new JavaBigDecimal(tokenValue);
                    }
                case SqlTokenType.BooleanLiteral:
                    {
                        return Token.ValueFor.TRUE.Equals(tokenValue,
                            InvariantCultureIgnoreCase) ? JavaBoolean.TRUE
                            : JavaBoolean.FALSE;
                    }
                case SqlTokenType.DateLiteral:
                    {
                        return HsqlDateTime.dateValue(tokenValue);
                    }
                case SqlTokenType.TimeLiteral:
                    {
                        return HsqlDateTime.timeValue(tokenValue);
                    }
                case SqlTokenType.TimestampLiteral:
                    {
                        return HsqlDateTime.timestampValue(tokenValue);
                    }
                default:
                    {
                        return tokenValue;
                    }
            }
        }