Пример #1
0
        /// <summary>
        /// tokenizes until it can't anymore
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="rawData"></param>
        /// <param name="state"></param>
        /// <param name="newPosition"></param>
        /// <returns></returns>
        public static List <IToken <T> > ForwardMovingTokenize <T>(this T[] rawData, object state, IForwardMovingTokenizer <T> parser, out int newPosition)
        {
            Debug.WriteLine(string.Format("ForwardMovingTokenize. starts {0}", string.Join("", rawData)));

            List <IToken <T> > rv = new List <IToken <T> >();

            newPosition = 0;

            if (rawData == null)
            {
                return(rv);
            }

            if (parser == null)
            {
                return(rv);
            }

            int        pos    = 0;
            int        maxPos = rawData.Length - 1;
            IToken <T> token  = null;
            IForwardMovingTokenizer <T> currentParser = parser;
            IForwardMovingTokenizer <T> lastParser    = null;
            bool goodParse = true;

            int counter = 0;

            while (goodParse && currentParser != null && pos > -1 && pos <= maxPos)
            {
                if (currentParser != null)
                {
                    lastParser = currentParser;
                }

                counter++;

                var    priorToken        = token;
                var    startPos          = pos;
                string currentParserId   = currentParser.As <IHasId>(false).With(x => x.Id.ToString());
                string currentParserType = currentParser.GetType().Name;

                Debug.WriteLine("ForwardMovingTokenize. iteration starts @ {0} with {1}, {2}", pos, currentParserId, currentParserType);
                goodParse = currentParser.Parse(rawData, pos, state, token, out pos, out token, out currentParser);
                if (goodParse)
                {
                    Debug.WriteLine("ForwardMovingTokenize. iteration success @ {0} with {1} ", pos, currentParser.As <IHasId>(false).With(x => x.Id.ToString()));

                    if (token != null)
                    {
                        token.PriorToken = priorToken;
                        //decorate token with positional info
                        token = token.HasStartEnd(startPos, pos);

                        Debug.WriteLine(string.Format("ForwardMovingTokenize. Token={0}", token.DumpToken()));

                        rv.Add(token);
                    }
                }
                else
                {
                    Debug.WriteLine("ForwardMovingTokenize. iteration fail @ {0} with {1}", pos, currentParser.As <IHasId>(false).With(x => x.Id.ToString()));
                }
            }

            newPosition = pos;

            Debug.WriteLine(string.Format("ForwardMovingTokenize. ends {0}", string.Join("", rawData)));

            return(rv);
        }