public void ParseCaseSensitiveTokens()
        {
            var tokenizer = new Tokenizer();

            tokenizer.AddToken("Roger", true, true, TestTag);
            tokenizer.AddToken("Alsing", true, true, TestTag);

            const string text = @"Roger ROGER rogeR Alsing ALSING __Roger__ __Alsing__";
            //                   "XXXXX             XXXXXX                            "

            // only two tokens should be found, the rest
            // are either wrong casing or do not have separators next to them

            tokenizer.Text = text;
            var tokens = tokenizer.Tokenize();

            var testTokens = from token in tokens
                             where token.HasTag(TestTag)
                             select token;

            Assert.AreEqual(2, testTokens.ToList().Count);
        }
        public void ParsePatternWithoutSeparators()
        {
            var tokenizer = new Tokenizer();

            tokenizer.AddPattern(IntPatternMatcher.Default, true, false, TestTag);

            tokenizer.AddToken("Alsing", false, true, TestTag);

            const string text = @"The quick brown 1337 fox jumped 0v3r the little pig";
            //                                    XXXX            X X

            // only two tokens should be found, the rest
            // are either wrong casing or do not have separators next to them

            tokenizer.Text = text;
            var tokens = tokenizer.Tokenize();

            var testTokens = from token in tokens
                             where token.HasTag(TestTag)
                             select token;

            Assert.AreEqual(3, testTokens.ToList().Count);
        }