public void ParseCaseSensitiveTokens()
{
var tokenizer = new Tokenizer();
tokenizer.AddToken("Roger", true, true, TestTag);
tokenizer.AddToken("Alsing", true, true, TestTag);
const string text = @"Roger ROGER rogeR Alsing ALSING __Roger__ __Alsing__";
// "XXXXX XXXXXX "
// only two tokens should be found, the rest
// are either wrong casing or do not have separators next to them
tokenizer.Text = text;
var tokens = tokenizer.Tokenize();
var testTokens = from token in tokens
where token.HasTag(TestTag)
select token;
Assert.AreEqual(2, testTokens.ToList().Count);
}
public void ParsePatternWithoutSeparators()
{
var tokenizer = new Tokenizer();
tokenizer.AddPattern(IntPatternMatcher.Default, true, false, TestTag);
tokenizer.AddToken("Alsing", false, true, TestTag);
const string text = @"The quick brown 1337 fox jumped 0v3r the little pig";
// XXXX X X
// only two tokens should be found, the rest
// are either wrong casing or do not have separators next to them
tokenizer.Text = text;
var tokens = tokenizer.Tokenize();
var testTokens = from token in tokens
where token.HasTag(TestTag)
select token;
Assert.AreEqual(3, testTokens.ToList().Count);
}