public void TestUnigrams()
{
NGramTokenFilter filter = new NGramTokenFilter(input, 1, 1);
AssertTokenStreamContents(filter, new String[] { "a", "b", "c", "d", "e" }, new int[] { 0, 1, 2, 3, 4 }, new int[] { 1, 2, 3, 4, 5 });
}
public void TestBigrams()
{
NGramTokenFilter filter = new NGramTokenFilter(input, 2, 2);
AssertTokenStreamContents(filter, new String[] { "ab", "bc", "cd", "de" }, new int[] { 0, 1, 2, 3 }, new int[] { 2, 3, 4, 5 });
}
public void TestOversizedNgrams()
{
NGramTokenFilter filter = new NGramTokenFilter(input, 6, 7);
AssertTokenStreamContents(filter, new String[0], new int[0], new int[0]);
}
