public void ExtractTokensFromString()
{
var analyzer = new StandardAnalyzer(Version.LUCENE_29);
var tokens = analyzer.Tokenize("notepad2 note2 note2 no2 n2 n2 n2 n2 n2 n2 note2 notepad2");
Assert.Equal(12, tokens.Count());
var deduplicatedTokens = new HashSet<string>(tokens);
Assert.Equal(4, deduplicatedTokens.Count);
Assert.Contains("notepad2", deduplicatedTokens);
Assert.Contains("note2", deduplicatedTokens);
Assert.Contains("no2", deduplicatedTokens);
Assert.Contains("n2", deduplicatedTokens);
}
