public void AddPattern(string prefix, bool caseSensitive, bool needSeparators, IPatternMatcher matcher,
                               object[] tags)
        {
            if (string.IsNullOrEmpty(prefix))
            {
                throw new ArgumentNullException("prefix");
            }

            TokenTreeNode node = AddTokenInternal(prefix, caseSensitive);

            var patternMatcherReference = new PatternMatchReference(matcher)
            {
                NextSibling    = FirstExpression,
                Tags           = tags,
                NeedSeparators = needSeparators
            };

            node.FirstExpression = patternMatcherReference;
        }
        public TokenTreeNode AddTokenInternal(string token, bool caseSensitive)
        {
            Char = token[0];


            if (!caseSensitive)
            {
                ContainsCaseInsensitiveData = true;
            }

            if (token.Length == 1)
            {
                return(this);
            }

            string leftovers  = token.Substring(1);
            char   childChar  = leftovers[0];
            int    childIndex = childChar & 0xff;
            //make a lookupindex (dont mind if unicode chars end up as siblings as ascii)

            TokenTreeNode node = ChildNodes[childIndex];
            TokenTreeNode res;

            if (node == null)
            {
                var child = new TokenTreeNode();
                ChildNodes[childIndex] = child;
                res = child.AddTokenInternal(leftovers, caseSensitive);

                MakeRepeatingWS(child);
            }
            else
            {
                node = GetMatchingNode(childChar, node);
                res  = node.AddTokenInternal(leftovers, caseSensitive);
            }

            return(res);
        }