public virtual void TestMockGraphTokenFilterAfterHoles()
{
for (int iter = 0; iter < 100 * RandomMultiplier; iter++)
{
if (Verbose)
{
Console.WriteLine("\nTEST: iter=" + iter);
}
// Make new analyzer each time, because MGTF has fixed
// seed:
Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream t2 = new RemoveATokens(t);
TokenStream t3 = new MockGraphTokenFilter(Random, t2);
return(new TokenStreamComponents(t, t3));
});
Random random = Random;
CheckAnalysisConsistency(random, a, false, "a b c d e f g h i j k");
CheckAnalysisConsistency(random, a, false, "x y a b c d e f g h i j k");
CheckAnalysisConsistency(random, a, false, "a b c d e f g h i j k a");
CheckAnalysisConsistency(random, a, false, "a b c d e f g h i j k a x y");
}
}
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream t2 = new MockGraphTokenFilter(Random(), t);
return(new TokenStreamComponents(t, t2));
}
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer t = new GraphTokenizer(reader);
TokenStream t2 = new MockGraphTokenFilter(Random(), t);
return(new TokenStreamComponents(t, t2));
}
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream t1 = new MockHoleInjectingTokenFilter(Random, t);
TokenStream t2 = new MockGraphTokenFilter(Random, t1);
return(new TokenStreamComponents(t, t2));
}
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream t2 = new RemoveATokens(t);
TokenStream t3 = new MockGraphTokenFilter(Random(), t2);
return(new TokenStreamComponents(t, t3));
}
public virtual void TestMockGraphTokenFilterOnGraphInput()
{
for (int iter = 0; iter < 100 * RandomMultiplier; iter++)
{
if (Verbose)
{
Console.WriteLine("\nTEST: iter=" + iter);
}
// Make new analyzer each time, because MGTF has fixed
// seed:
Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer t = new GraphTokenizer(reader);
TokenStream t2 = new MockGraphTokenFilter(Random, t);
return(new TokenStreamComponents(t, t2));
});
CheckAnalysisConsistency(Random, a, false, "a/x:3 c/y:2 d e f/z:4 g h i j k");
}
}
public virtual void TestMockGraphTokenFilterRandom()
{
for (int iter = 0; iter < 10 * RandomMultiplier; iter++)
{
if (Verbose)
{
Console.WriteLine("\nTEST: iter=" + iter);
}
// Make new analyzer each time, because MGTF has fixed
// seed:
Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream t2 = new MockGraphTokenFilter(Random, t);
return(new TokenStreamComponents(t, t2));
});
Random random = Random;
CheckRandomData(random, a, 5, AtLeast(100));
}
}
