// Move to the next token
        // This updates the values in the token instance and returns true if successful.
        internal bool MoveNext()
        {
            // Hold onto the ID of the last token.
            // It will be needed by some of the special cases.
            this.previousID = this.token.TokenID;

            // If there are no more characters, we can't get another token.
            if (!AdvanceChar())
            {
                return(false);
            }

            if (XPathCharTypes.IsNCNameStart(this.ch))
            {
                // Extract a QName if we've got the start of an NCName
                TokenizeQName();
            }
            else if (XPathCharTypes.IsDigit(this.ch))
            {
                // Extract a number
                TokenizeNumber();
            }
            else
            {
                // Everything else is a single/double character token, or a variable.
                switch (this.ch)
                {
                case '(':
                    token.Set(XPathTokenID.LParen);
                    break;

                case ')':
                    token.Set(XPathTokenID.RParen);
                    break;

                case '[':
                    token.Set(XPathTokenID.LBracket);
                    break;

                case ']':
                    token.Set(XPathTokenID.RBracket);
                    break;

                case '.':
                    // Watch for a double period
                    if (PeekChar() == '.')
                    {
                        AdvanceChar();
                        token.Set(XPathTokenID.DblPeriod);
                    }
                    else
                    {
                        // Check if the period is the start of a number
                        if (XPathCharTypes.IsDigit(PeekChar()))
                        {
                            TokenizeNumber();
                        }
                        else
                        {
                            token.Set(XPathTokenID.Period);
                        }
                    }
                    break;

                case '@':
                    token.Set(XPathTokenID.AtSign);
                    break;

                case ',':
                    token.Set(XPathTokenID.Comma);
                    break;

                case ':':
                    // Only a double colon is permitted.
                    // The single colon part of the QName is consumed in TokenizeQName if it is valid
                    if (PeekChar() == ':')
                    {
                        AdvanceChar();
                        token.Set(XPathTokenID.DblColon);
                    }
                    else
                    {
                        ThrowError(QueryCompileError.UnexpectedToken, CurrentSubstring());
                    }
                    break;



                case '/':
                    // Check for a double slash
                    if (PeekChar() == '/')
                    {
                        AdvanceChar();
                        token.Set(XPathTokenID.DblSlash);
                    }
                    else
                    {
                        token.Set(XPathTokenID.Slash);
                    }
                    break;

                case '|':
                    token.Set(XPathTokenID.Pipe);
                    break;

                case '+':
                    token.Set(XPathTokenID.Plus);
                    break;

                case '-':
                    token.Set(XPathTokenID.Minus);
                    break;

                case '=':
                    token.Set(XPathTokenID.Eq);
                    break;

                case '!':
                    // This can only be the start of a '!='
                    // 'not' is a negation in XPath
                    if (PeekChar() == '=')
                    {
                        AdvanceChar();
                        token.Set(XPathTokenID.Neq);
                    }
                    else
                    {
                        ThrowError(QueryCompileError.UnsupportedOperator, CurrentSubstring());
                    }
                    break;

                case '<':
                    // Watch for '<='
                    if (PeekChar() == '=')
                    {
                        AdvanceChar();
                        token.Set(XPathTokenID.Lte);
                    }
                    else
                    {
                        token.Set(XPathTokenID.Lt);
                    }
                    break;

                case '>':
                    // Watch for '>='
                    if (PeekChar() == '=')
                    {
                        AdvanceChar();
                        token.Set(XPathTokenID.Gte);
                    }
                    else
                    {
                        token.Set(XPathTokenID.Gt);
                    }
                    break;


                case '*':
                    // Check if we're supposed to parse a '*' as a multiply
                    if (IsSpecialPrev())
                    {
                        token.Set(XPathTokenID.Multiply);
                    }
                    else
                    {
                        token.Set(XPathTokenID.Wildcard, new XPathParser.QName(string.Empty, QueryDataModel.Wildcard));
                    }
                    break;

                case '$':
                    // Make sure '$' was followed by something that counts as a variable name
                    XPathParser.QName qname = GetQName();
                    if (qname.Prefix.Length == 0 && qname.Name.Length == 0)
                    {
                        AdvanceChar();
                        ThrowError(QueryCompileError.InvalidVariable, this.ch == char.MinValue ? string.Empty : CurrentSubstring());
                    }
                    token.Set(XPathTokenID.Variable, qname);
                    break;

                case '\"':
                    TokenizeLiteral('\"');
                    break;

                case '\'':
                    TokenizeLiteral('\'');
                    break;

                default:
                    // Unrecognized character
                    token.Set(XPathTokenID.Unknown);
                    break;
                }
            }

            // Whitespace can mark the end of a token, but is not part of the XPath syntax
            ConsumeWhitespace();

            return(true);
        }