public virtual void TestMockGraphTokenFilterAfterHoles() { for (int iter = 0; iter < 100 * RandomMultiplier; iter++) { if (Verbose) { Console.WriteLine("\nTEST: iter=" + iter); } // Make new analyzer each time, because MGTF has fixed // seed: Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => { Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); TokenStream t2 = new RemoveATokens(t); TokenStream t3 = new MockGraphTokenFilter(Random, t2); return(new TokenStreamComponents(t, t3)); }); Random random = Random; CheckAnalysisConsistency(random, a, false, "a b c d e f g h i j k"); CheckAnalysisConsistency(random, a, false, "x y a b c d e f g h i j k"); CheckAnalysisConsistency(random, a, false, "a b c d e f g h i j k a"); CheckAnalysisConsistency(random, a, false, "a b c d e f g h i j k a x y"); } }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); TokenStream t2 = new MockGraphTokenFilter(Random(), t); return(new TokenStreamComponents(t, t2)); }
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new GraphTokenizer(reader); TokenStream t2 = new MockGraphTokenFilter(Random(), t); return(new TokenStreamComponents(t, t2)); }
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); TokenStream t1 = new MockHoleInjectingTokenFilter(Random, t); TokenStream t2 = new MockGraphTokenFilter(Random, t1); return(new TokenStreamComponents(t, t2)); }
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); TokenStream t2 = new RemoveATokens(t); TokenStream t3 = new MockGraphTokenFilter(Random(), t2); return(new TokenStreamComponents(t, t3)); }
public virtual void TestMockGraphTokenFilterOnGraphInput() { for (int iter = 0; iter < 100 * RandomMultiplier; iter++) { if (Verbose) { Console.WriteLine("\nTEST: iter=" + iter); } // Make new analyzer each time, because MGTF has fixed // seed: Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => { Tokenizer t = new GraphTokenizer(reader); TokenStream t2 = new MockGraphTokenFilter(Random, t); return(new TokenStreamComponents(t, t2)); }); CheckAnalysisConsistency(Random, a, false, "a/x:3 c/y:2 d e f/z:4 g h i j k"); } }
public virtual void TestMockGraphTokenFilterRandom() { for (int iter = 0; iter < 10 * RandomMultiplier; iter++) { if (Verbose) { Console.WriteLine("\nTEST: iter=" + iter); } // Make new analyzer each time, because MGTF has fixed // seed: Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => { Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); TokenStream t2 = new MockGraphTokenFilter(Random, t); return(new TokenStreamComponents(t, t2)); }); Random random = Random; CheckRandomData(random, a, 5, AtLeast(100)); } }