public virtual void TestTokenStream() { QueryAutoStopWordAnalyzer a = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false), reader, 10); TokenStream ts = a.GetTokenStream("repetitiveField", "this boring"); AssertTokenStreamContents(ts, new string[] { "this" }); }
public virtual void TestNoStopwords() { // Note: an empty list of fields passed in protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Enumerable.Empty<string>(), 1); TokenStream protectedTokenStream = protectedAnalyzer.TokenStream("variedField", "quick"); AssertTokenStreamContents(protectedTokenStream, new string[] { "quick" }); protectedTokenStream = protectedAnalyzer.TokenStream("repetitiveField", "boring"); AssertTokenStreamContents(protectedTokenStream, new string[] { "boring" }); }
public virtual void TestStopwordsPerFieldMaxDocFreq() { protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, new string[] { "repetitiveField" }, 10); int numStopWords = protectedAnalyzer.GetStopWords("repetitiveField").Length; assertTrue("Should have identified stop words", numStopWords > 0); protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, new string[] { "repetitiveField", "variedField" }, 10); int numNewStopWords = protectedAnalyzer.GetStopWords("repetitiveField").Length + protectedAnalyzer.GetStopWords("variedField").Length; assertTrue("Should have identified more stop words", numNewStopWords > numStopWords); }
public virtual void TestNoFieldNamePollution() { protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, new string[] { "repetitiveField" }, 10); TokenStream protectedTokenStream = protectedAnalyzer.GetTokenStream("repetitiveField", "boring"); // Check filter set up OK AssertTokenStreamContents(protectedTokenStream, new string[0]); protectedTokenStream = protectedAnalyzer.GetTokenStream("variedField", "boring"); // Filter should not prevent stopwords in one field being used in another AssertTokenStreamContents(protectedTokenStream, new string[] { "boring" }); }
public virtual void TestStopwordsPerFieldMaxPercentDocs() { protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, new string[] { "variedField" }, 1f / 2f); TokenStream protectedTokenStream = protectedAnalyzer.GetTokenStream("repetitiveField", "boring"); // A filter on one Field should not affect queries on another AssertTokenStreamContents(protectedTokenStream, new string[] { "boring" }); protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, new string[] { "variedField", "repetitiveField" }, 1f / 2f); protectedTokenStream = protectedAnalyzer.GetTokenStream("repetitiveField", "boring"); // A filter on the right Field should affect queries on it AssertTokenStreamContents(protectedTokenStream, new string[0]); }
public virtual void TestStopwordsAllFieldsMaxPercentDocs() { protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, 1f / 2f); TokenStream protectedTokenStream = protectedAnalyzer.TokenStream("repetitiveField", "boring"); // A filter on terms in > one half of docs remove boring AssertTokenStreamContents(protectedTokenStream, new string[0]); protectedTokenStream = protectedAnalyzer.TokenStream("repetitiveField", "vaguelyboring"); // A filter on terms in > half of docs should not remove vaguelyBoring AssertTokenStreamContents(protectedTokenStream, new string[] { "vaguelyboring" }); protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, 1f / 4f); protectedTokenStream = protectedAnalyzer.TokenStream("repetitiveField", "vaguelyboring"); // A filter on terms in > quarter of docs should remove vaguelyBoring AssertTokenStreamContents(protectedTokenStream, new string[0]); }
public override void SetUp() { dir = new RAMDirectory(); appAnalyzer = new WhitespaceAnalyzer(); IndexWriter writer = new IndexWriter(dir, appAnalyzer, true, IndexWriter.MaxFieldLength.UNLIMITED); int numDocs = 200; for (int i = 0; i < numDocs; i++) { Document doc = new Document(); String variedFieldValue = variedFieldValues[i % variedFieldValues.Length]; String repetitiveFieldValue = repetitiveFieldValues[i % repetitiveFieldValues.Length]; doc.Add(new Field("variedField", variedFieldValue, Field.Store.YES, Field.Index.ANALYZED)); doc.Add(new Field("repetitiveField", repetitiveFieldValue, Field.Store.YES, Field.Index.ANALYZED)); writer.AddDocument(doc); } writer.Close(); reader = IndexReader.Open(dir, true); protectedAnalyzer = new QueryAutoStopWordAnalyzer(Version.LUCENE_CURRENT, appAnalyzer); base.SetUp(); }
public virtual void TestDefaultStopwordsAllFields() { protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader); TokenStream protectedTokenStream = protectedAnalyzer.TokenStream("repetitiveField", "boring"); AssertTokenStreamContents(protectedTokenStream, new string[0]); // Default stop word filtering will remove boring }
public virtual void TestTokenStream() { QueryAutoStopWordAnalyzer a = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false), reader, 10); TokenStream ts = a.TokenStream("repetitiveField", "this boring"); AssertTokenStreamContents(ts, new string[] { "this" }); }
public virtual void TestNoFieldNamePollution() { protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.AsList("repetitiveField"), 10); TokenStream protectedTokenStream = protectedAnalyzer.TokenStream("repetitiveField", "boring"); // Check filter set up OK AssertTokenStreamContents(protectedTokenStream, new string[0]); protectedTokenStream = protectedAnalyzer.TokenStream("variedField", "boring"); // Filter should not prevent stopwords in one field being used in another AssertTokenStreamContents(protectedTokenStream, new string[] { "boring" }); }
public virtual void TestStopwordsPerFieldMaxDocFreq() { protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.AsList("repetitiveField"), 10); int numStopWords = protectedAnalyzer.GetStopWords("repetitiveField").Length; assertTrue("Should have identified stop words", numStopWords > 0); protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.AsList("repetitiveField", "variedField"), 10); int numNewStopWords = protectedAnalyzer.GetStopWords("repetitiveField").Length + protectedAnalyzer.GetStopWords("variedField").Length; assertTrue("Should have identified more stop words", numNewStopWords > numStopWords); }
public virtual void TestStopwordsPerFieldMaxPercentDocs() { protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.AsList("variedField"), 1f / 2f); TokenStream protectedTokenStream = protectedAnalyzer.TokenStream("repetitiveField", "boring"); // A filter on one Field should not affect queries on another AssertTokenStreamContents(protectedTokenStream, new string[] { "boring" }); protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.AsList("variedField", "repetitiveField"), 1f / 2f); protectedTokenStream = protectedAnalyzer.TokenStream("repetitiveField", "boring"); // A filter on the right Field should affect queries on it AssertTokenStreamContents(protectedTokenStream, new string[0]); }
public void TestTokenStream() { QueryAutoStopWordAnalyzer a = new QueryAutoStopWordAnalyzer(Version.LUCENE_CURRENT, new WhitespaceAnalyzer()); a.AddStopWords(reader, 10); TokenStream ts = a.TokenStream("repetitiveField", new StringReader("this boring")); ITermAttribute termAtt = ts.GetAttribute<ITermAttribute>(); Assert.True(ts.IncrementToken()); Assert.AreEqual("this", termAtt.Term); Assert.False(ts.IncrementToken()); }
public void TestWrappingNonReusableAnalyzer() { QueryAutoStopWordAnalyzer a = new QueryAutoStopWordAnalyzer(Version.LUCENE_CURRENT, new NonreusableAnalyzer()); a.AddStopWords(reader, 10); int numHits = Search(a, "repetitiveField:boring"); Assert.True(numHits == 0); numHits = Search(a, "repetitiveField:vaguelyboring"); Assert.True(numHits == 0); }