Match() public method

public Match ( string text, int startIndex ) : MatchResult
text string
startIndex int
return MatchResult
TokenTree Class Documentation
Beispiel #1
0
        public Token[] Tokenize()
        {
            if (Text == null)
            {
                throw new ArgumentNullException("Text");
            }

            MakeImmutable();

            var tokens = new List <Token>();

            int index = 0;

            while (index < Text.Length)
            {
                MatchResult match = tree.Match(Text, index);

                if (match.Found)
                {
                    string dummyText  = Text.Substring(index, match.Index - index);
                    var    dummyToken = new Token(dummyText, null);
                    tokens.Add(dummyToken);

                    var realToken = new Token(match.GetText(), match.Tags);
                    index = match.Index + match.Length;
                    tokens.Add(realToken);
                }
                else
                {
                    string dummyText  = Text.Substring(index);
                    var    dummyToken = new Token(dummyText, null);
                    tokens.Add(dummyToken);

                    index = Text.Length;
                }
            }

            return(tokens.ToArray());
        }