Exemple #1
0
        private void ChangeState(ITokenizer tokenizer)
        {
            // don't add closing quote
            tokenizer.StrmReader.Read();

            IState nextState = NewToken.Instance();

            nextState.Read(tokenizer);
        }
Exemple #2
0
 /// <summary>
 /// Changes the state.
 /// </summary>
 /// <param name="tokenizer">The tokenizer.</param>
 private void ChangeState(ITokenizer tokenizer)
 {
     if (this.ReadCharsAreValidIdentifier(this.TokenCharacters.ToString()))
     {
         IState nextState = NewToken.Instance();
         nextState.Read(tokenizer);
     }
     else
     {
         throw new Exception("Invalid identifier");
     }
 }
Exemple #3
0
        /// <summary>
        /// Changes the state of the ITokenizer instance.
        /// </summary>
        /// <param name="tokenizer">The tokenizer.</param>
        private void ChangeState(ITokenizer tokenizer)
        {
            IState nextState = null;

            if (this.ReadCharsAreValidKeyword(tokenizer))
            {
                tokenizer.Tokens.Add(this.CreateTokenObject());
                nextState = NewToken.Instance();
            }
            else
            {
                nextState = Identifier.Instance(this.TokenCharacters);
            }

            nextState.Read(tokenizer);
        }
Exemple #4
0
        /// <summary>
        /// Changes the state.
        /// </summary>
        /// <param name="tokenizer">The tokenizer.</param>
        private void ChangeState(ITokenizer tokenizer)
        {
            //if (new StackTrace().FrameCount > 1700)
            //    throw new Exception("sdfgdfgdgdfgdfgdfgdfg");
            StreamReader streamReader = tokenizer.StrmReader;

            IState nextState = null;

            while (!streamReader.EndOfStream)
            {
                char peekedChar = (char)streamReader.Peek();

                //  the order of these ifs matters -
                //  IsIntegerConstant must be checked before Identifier.
                if (this.IsPossibleKeyword(peekedChar))
                {
                    nextState = Keyword.Instance();
                }
                else if (this.IsWhiteSpace(peekedChar))
                {
                    tokenizer.StrmReader.Read();
                    nextState = NewToken.Instance();
                }
                else if (this.IsSymbol(peekedChar))
                {
                    nextState = Symbol.Instance();
                }
                else if (this.IsIntegerConstant(peekedChar))
                {
                    nextState = IntegerConstant.Instance();
                }
                else if (this.IsPossibleIdentifierCharacter(peekedChar))
                {
                    nextState = Identifier.Instance();
                }

                // nasty hack to get over stackoverflow - don't call recusive on each new token
                if (nextState != null && nextState.GetType() != typeof(NewToken))
                {
                    nextState.Read(tokenizer);
                }
            }
        }
Exemple #5
0
        /// <summary>
        /// Changes the state.
        /// </summary>
        /// <param name="tokenizer">The tokenizer.</param>
        private void ChangeState(ITokenizer tokenizer)
        {
            StreamReader streamReader = tokenizer.StrmReader;

            IState nextState = null;

            if (!streamReader.EndOfStream)
            {
                char peekedChar = (char)streamReader.Peek();

                //  the order of these ifs matters -
                //  IsIntegerConstant must be checked before Identifier.
                if (this.IsPossibleKeyword(peekedChar))
                {
                    nextState = Keyword.Instance();
                }
                else if (this.IsWhiteSpace(peekedChar))
                {
                    tokenizer.StrmReader.Read();
                    nextState = NewToken.Instance();
                }
                else if (this.IsSymbol(peekedChar))
                {
                    nextState = Symbol.Instance();
                }
                else if (this.IsIntegerConstant(peekedChar))
                {
                    nextState = IntegerConstant.Instance();
                }
                else if (this.IsPossibleIdentifierCharacter(peekedChar))
                {
                    nextState = Identifier.Instance();
                }

                if (nextState != null)
                {
                    nextState.Read(tokenizer);
                }
            }
        }
Exemple #6
0
        /// <summary>
        /// Changes the state of the tokenizer
        /// </summary>
        /// <param name="tokenizer">The tokenizer.</param>
        /// <param name="symbolCharacter">The symbol character.</param>
        private void ChangeState(ITokenizer tokenizer, char symbolChar)
        {
            this.CreateTokenObject();

            IState nextState = null;

            if (symbolChar == '"')
            {
                nextState = StringConstant.Instance();
            }
            else if (this.IsComment(tokenizer.StrmReader, symbolChar))
            {
                //pass the starting comment symbol to Comment instance
                nextState = Comment.Instance(tokenCharacters);
            }
            else
            {
                // else its just a symbol so create the token.
                tokenizer.Tokens.Add(this.CreateTokenObject());
                nextState = NewToken.Instance();
            }

            nextState.Read(tokenizer);
        }
        private void ChangeState(ITokenizer tokenizer)
        {
            IState nextState = NewToken.Instance();

            nextState.Read(tokenizer);
        }