public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
TokenStream tokenStream = new MockGraphTokenFilter(Random(), tokenizer);
return(new TokenStreamComponents(tokenizer, tokenStream));
}
public virtual void TestRandom2GraphAfter()
{
int numIters = AtLeast(3);
Random random = Random;
for (int i = 0; i < numIters; i++)
{
b = new SynonymMap.Builder(random.nextBoolean());
int numEntries = AtLeast(10);
for (int j = 0; j < numEntries; j++)
{
Add(RandomNonEmptyString(), RandomNonEmptyString(), random.nextBoolean());
}
SynonymMap map = b.Build();
bool ignoreCase = random.nextBoolean();
Analyzer analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
TokenStream syns = new SynonymFilter(tokenizer, map, ignoreCase);
TokenStream graph = new MockGraphTokenFilter(Random, syns);
return(new TokenStreamComponents(tokenizer, graph));
});
CheckRandomData(random, analyzer, 100);
}
}
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
TokenStream syns = new SynonymFilter(tokenizer, map, ignoreCase);
TokenStream graph = new MockGraphTokenFilter(Random(), syns);
return(new TokenStreamComponents(tokenizer, graph));
}
public virtual void TestRandomHugeStringsGraphAfter()
{
Random random = Random;
CheckRandomData(random,
Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
TokenStream tokenStream = new MockGraphTokenFilter(Random, tokenizer);
return(new TokenStreamComponents(tokenizer, tokenStream));
})
, 100 * RandomMultiplier, 8192);
}
public void TestRandomHugeStringsMockGraphAfter()
{
// Randomly inject graph tokens after JapaneseTokenizer:
Random random = Random;
CheckRandomData(random,
Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer tokenizer = new JapaneseTokenizer(reader, ReadDict(), false, JapaneseTokenizerMode.SEARCH);
TokenStream graph = new MockGraphTokenFilter(Random, tokenizer);
return(new TokenStreamComponents(tokenizer, graph));
}),
100 * RandomMultiplier, 8192);
}
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
TokenStream tokenStream = new MockGraphTokenFilter(Random(), tokenizer);
return new TokenStreamComponents(tokenizer, tokenStream);
}
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream t1 = new MockHoleInjectingTokenFilter(Random(), t);
TokenStream t2 = new MockGraphTokenFilter(Random(), t1);
return new TokenStreamComponents(t, t2);
}
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer t = new GraphTokenizer(reader);
TokenStream t2 = new MockGraphTokenFilter(Random(), t);
return new TokenStreamComponents(t, t2);
}
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream t2 = new RemoveATokens(t);
TokenStream t3 = new MockGraphTokenFilter(Random(), t2);
return new TokenStreamComponents(t, t3);
}
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
TokenStream syns = new SynonymFilter(tokenizer, map, ignoreCase);
TokenStream graph = new MockGraphTokenFilter(Random(), syns);
return new TokenStreamComponents(tokenizer, graph);
}
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
TokenStream tokenStream = new MockGraphTokenFilter(random(), tokenizer);
return new TokenStreamComponents(tokenizer, tokenStream);
}
