public void TestEndNotStopWord() { CharArraySet stopWords = StopFilter.MakeStopSet(TEST_VERSION_CURRENT, "to"); TokenStream stream = new MockTokenizer(new StringReader("go to")); TokenStream filter = new SuggestStopFilter(stream, stopWords); AssertTokenStreamContents(filter, new string[] { "go", "to" }, new int[] { 0, 3 }, new int[] { 2, 5 }, null, new int[] { 1, 1 }, null, 5, new bool[] { false, true }, true); }
public void TestMidStopWord() { CharArraySet stopWords = StopFilter.MakeStopSet(TEST_VERSION_CURRENT, "to"); TokenStream stream = new MockTokenizer(new StringReader("go to school")); TokenStream filter = new SuggestStopFilter(stream, stopWords); filter = new SuggestStopFilter(stream, stopWords); AssertTokenStreamContents(filter, new String[] { "go", "school" }, new int[] { 0, 6 }, new int[] { 2, 12 }, null, new int[] { 1, 2 }, null, 12, new bool[] { false, false }, true); }
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); return(new TokenStreamComponents(tokenizer, tokenizer)); }
public override TokenStreamComponents CreateComponents(string fieldName, System.IO.TextReader reader) { MockTokenizer tokenizer = new MockTokenizer(reader); return new TokenStreamComponents(tokenizer, new MockSynonymFilter(tokenizer)); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); TokenStream stream = new SynonymFilter(tokenizer, map, ignoreCase); return new TokenStreamComponents(tokenizer, new RemoveDuplicatesTokenFilter(stream)); }
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); return(new TokenStreamComponents(result, new MockPayloadFilter(result, fieldName))); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); return new TokenStreamComponents(tokenizer, new SimplePayloadFilter(tokenizer)); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(reader, Dfa, Lowercase, Limit); return new TokenStreamComponents(t, t); }
public virtual void TestMixupMultiValued() { Directory dir = NewDirectory(); RandomIndexWriter writer = new RandomIndexWriter(Random(), dir); Document doc = new Document(); Field field = new TextField("field", "", Field.Store.NO); TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true); Assert.IsFalse(ts.HasAttribute<PayloadAttribute>()); field.TokenStream = ts; doc.Add(field); Field field2 = new TextField("field", "", Field.Store.NO); Token withPayload = new Token("withPayload", 0, 11); withPayload.Payload = new BytesRef("test"); ts = new CannedTokenStream(withPayload); Assert.IsTrue(ts.HasAttribute<IPayloadAttribute>()); field2.TokenStream = ts; doc.Add(field2); Field field3 = new TextField("field", "", Field.Store.NO); ts = new MockTokenizer(new StringReader("nopayload"), MockTokenizer.WHITESPACE, true); Assert.IsFalse(ts.HasAttribute<PayloadAttribute>()); field3.TokenStream = ts; doc.Add(field3); writer.AddDocument(doc); DirectoryReader reader = writer.Reader; SegmentReader sr = GetOnlySegmentReader(reader); DocsAndPositionsEnum de = sr.TermPositionsEnum(new Term("field", "withPayload")); de.NextDoc(); de.NextPosition(); Assert.AreEqual(new BytesRef("test"), de.Payload); writer.Dispose(); reader.Dispose(); dir.Dispose(); }
public void TestMockTokenizerCtor() { var sr = new StringReader("Hello"); var mt = new MockTokenizer(sr); }
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { PayloadData payload; FieldToData.TryGetValue(fieldName, out payload); Tokenizer ts = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); TokenStream tokenStream = (payload != null) ? (TokenStream)new PayloadFilter(ts, payload.Data, payload.Offset, payload.Length) : ts; return new TokenStreamComponents(ts, tokenStream); }
public virtual void TestMixupDocs() { Directory dir = NewDirectory(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, null); iwc.SetMergePolicy(NewLogMergePolicy()); RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc); Document doc = new Document(); Field field = new TextField("field", "", Field.Store.NO); TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true); Assert.IsFalse(ts.HasAttribute<PayloadAttribute>()); field.TokenStream = ts; doc.Add(field); writer.AddDocument(doc); Token withPayload = new Token("withPayload", 0, 11); withPayload.Payload = new BytesRef("test"); ts = new CannedTokenStream(withPayload); Assert.IsTrue(ts.HasAttribute<IPayloadAttribute>()); field.TokenStream = ts; writer.AddDocument(doc); ts = new MockTokenizer(new StringReader("another"), MockTokenizer.WHITESPACE, true); Assert.IsFalse(ts.HasAttribute<PayloadAttribute>()); field.TokenStream = ts; writer.AddDocument(doc); DirectoryReader reader = writer.Reader; AtomicReader sr = SlowCompositeReaderWrapper.Wrap(reader); DocsAndPositionsEnum de = sr.TermPositionsEnum(new Term("field", "withPayload")); de.NextDoc(); de.NextPosition(); Assert.AreEqual(new BytesRef("test"), de.Payload); writer.Dispose(); reader.Dispose(); dir.Dispose(); }
public void TestMockTokenizerCtor() { var sr = new StringReader("Hello"); var mt = new MockTokenizer(sr); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(reader, Dfa, Lowercase, Limit); return(new TokenStreamComponents(t, t)); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false, 5); return(new TokenStreamComponents(t, t)); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); TokenStream t2 = new RemoveATokens(t); TokenStream t3 = new MockGraphTokenFilter(Random(), t2); return new TokenStreamComponents(t, t3); }
public override TokenStreamComponents CreateComponents(string fieldName, System.IO.TextReader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); return new TokenStreamComponents(tokenizer, new MockCollationFilter(tokenizer)); }
/** Filters MockTokenizer with StopFilter. */ public override sealed TokenStreamComponents CreateComponents(String fieldName, TextReader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); return new TokenStreamComponents(tokenizer, new QPTestFilter(tokenizer)); }
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(factory, reader, MockTokenizer.KEYWORD, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH); return(new TokenStreamComponents(t)); }
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); TokenStream t1 = new MockHoleInjectingTokenFilter(Random(), t); TokenStream t2 = new MockGraphTokenFilter(Random(), t1); return new TokenStreamComponents(t, t2); }
public void TestMultipleStopWordsEnd2() { CharArraySet stopWords = StopFilter.MakeStopSet(TEST_VERSION_CURRENT, "to", "the", "a"); TokenStream stream = new MockTokenizer(new StringReader("go to a the ")); TokenStream filter = new SuggestStopFilter(stream, stopWords); filter = new SuggestStopFilter(stream, stopWords); AssertTokenStreamContents(filter, new String[] { "go" }, new int[] { 0 }, new int[] { 2 }, null, new int[] { 1 }, null, 12, new bool[] { false }, true); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false, 5); return new TokenStreamComponents(t, t); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer t = new MockTokenizer(Factory, reader, MockTokenizer.KEYWORD, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH); return new TokenStreamComponents(t); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); return new TokenStreamComponents(tokenizer, new MockFixedLengthPayloadFilter(Random(), tokenizer, Length)); }
public override TokenStreamComponents CreateComponents(string fieldName, System.IO.TextReader reader) { Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); return new TokenStreamComponents(result, new TestPosIncrementFilter(result)); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); return new TokenStreamComponents(tokenizer, new PayloadFilter(PayloadCount, tokenizer)); }
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); return new TokenStreamComponents(result, new MockPayloadFilter(result, fieldName)); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); return new TokenStreamComponents(result, new PayloadFilter(result, fieldName)); }
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); return new TokenStreamComponents(tokenizer, tokenizer); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { MockTokenizer tokenizer = new MockTokenizer(reader, RunAutomaton, LowerCase, MaxTokenLength_Renamed); tokenizer.EnableChecks = EnableChecks_Renamed; MockTokenFilter filt = new MockTokenFilter(tokenizer, Filter); return new TokenStreamComponents(tokenizer, MaybePayload(filt, fieldName)); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); return new TokenStreamComponents(tokenizer, new TokenFilterAnonymousInnerClassHelper(this, tokenizer)); }