public virtual void TestLetterAscii()
{
Random random = Random;
Analyzer left = new MockAnalyzer(random, jvmLetter, false);
Analyzer right = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
return(new TokenStreamComponents(tokenizer, tokenizer));
});
for (int i = 0; i < 1000; i++)
{
string s = TestUtil.RandomSimpleString(random);
assertEquals(s, left.GetTokenStream("foo", newStringReader(s)), right.GetTokenStream("foo", newStringReader(s)));
}
}
public virtual void TestLetterAsciiHuge()
{
Random random = Random;
int maxLength = 8192; // CharTokenizer.IO_BUFFER_SIZE*2
MockAnalyzer left = new MockAnalyzer(random, jvmLetter, false);
left.MaxTokenLength = 255; // match CharTokenizer's max token length
Analyzer right = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
return(new TokenStreamComponents(tokenizer, tokenizer));
});
int numIterations = AtLeast(50);
for (int i = 0; i < numIterations; i++)
{
string s = TestUtil.RandomSimpleString(random, maxLength);
assertEquals(s, left.GetTokenStream("foo", newStringReader(s)), right.GetTokenStream("foo", newStringReader(s)));
}
}
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
return(new TokenStreamComponents(tokenizer, tokenizer));
}
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
return new TokenStreamComponents(tokenizer, tokenizer);
}
public virtual void TestGraphs()
{
TokenStream tk = new LetterTokenizer(TEST_VERSION_CURRENT, new StringReader("abc d efgh ij klmno p q"));
tk = new ShingleFilter(tk);
tk = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tk, 7, 10);
AssertTokenStreamContents(tk, new string[] { "efgh ij", "ij klmn", "ij klmno", "klmno p" }, new int[] { 6, 11, 11, 14 }, new int[] { 13, 19, 19, 21 }, new int[] { 3, 1, 0, 1 }, new int[] { 2, 2, 2, 2 }, 23);
}