public virtual void TestBackUnigram()
{
#pragma warning disable 612, 618
Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, input, Lucene43EdgeNGramTokenizer.Side.BACK, 1, 1);
#pragma warning restore 612, 618
AssertTokenStreamContents(tokenizer, new string[] { "e" }, new int[] { 4 }, new int[] { 5 }, 5); // abcde
}
public virtual void TestBackRangeOfNgrams()
{
#pragma warning disable 612, 618
Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, input, Lucene43EdgeNGramTokenizer.Side.BACK, 1, 3);
#pragma warning restore 612, 618
AssertTokenStreamContents(tokenizer, new string[] { "e", "de", "cde" }, new int[] { 4, 3, 2 }, new int[] { 5, 5, 5 }, null, null, null, 5, false); // abcde
}
protected internal override TokenStreamComponents CreateComponents(string fieldName, Reader reader)
{
#pragma warning disable 612, 618
Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, reader, Lucene43EdgeNGramTokenizer.Side.BACK, 2, 4);
#pragma warning restore 612, 618
return(new TokenStreamComponents(tokenizer, tokenizer));
}
public virtual void TestRandomStrings()
{
for (int i = 0; i < 10; i++)
{
int min = TestUtil.NextInt32(Random, 2, 10);
int max = TestUtil.NextInt32(Random, min, 20);
Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, min, max);
return(new TokenStreamComponents(tokenizer, tokenizer));
});
CheckRandomData(Random, a, 100 * RandomMultiplier, 20);
CheckRandomData(Random, a, 10 * RandomMultiplier, 8192);
}
Analyzer b = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
#pragma warning disable 612, 618
Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, reader, Lucene43EdgeNGramTokenizer.Side.BACK, 2, 4);
#pragma warning restore 612, 618
return(new TokenStreamComponents(tokenizer, tokenizer));
});
CheckRandomData(Random, b, 1000 * RandomMultiplier, 20, false, false);
CheckRandomData(Random, b, 100 * RandomMultiplier, 8192, false, false);
}
public virtual void TestTokenizerPositions()
{
#pragma warning disable 612, 618
Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, input, Lucene43EdgeNGramTokenizer.Side.FRONT, 1, 3);
#pragma warning restore 612, 618
AssertTokenStreamContents(tokenizer, new string[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, null, new int[] { 1, 0, 0 }, null, null, false);
tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"), 1, 3);
AssertTokenStreamContents(tokenizer, new string[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, null, new int[] { 1, 1, 1 }, null, null, false);
}