public override void TearDown() { Iw.Dispose(); TestUtil.CheckIndex(Dir); // for some extra coverage, checkIndex before we forceMerge Iwc.SetOpenMode(IndexWriterConfig.OpenMode_e.APPEND); IndexWriter iw = new IndexWriter(Dir, (IndexWriterConfig)Iwc.Clone()); iw.ForceMerge(1); iw.Dispose(); Dir.Dispose(); // just force a checkindex for now base.TearDown(); }
public virtual void SearchFiltered(IndexWriter writer, Directory directory, Filter filter, bool fullMerge) { for (int i = 0; i < 60; i++) //Simple docs { Document doc = new Document(); doc.Add(NewStringField(FIELD, Convert.ToString(i), Field.Store.YES)); writer.AddDocument(doc); } if (fullMerge) { writer.ForceMerge(1); } writer.Dispose(); BooleanQuery booleanQuery = new BooleanQuery(); booleanQuery.Add(new TermQuery(new Term(FIELD, "36")), BooleanClause.Occur.SHOULD); IndexReader reader = DirectoryReader.Open(directory); IndexSearcher indexSearcher = NewSearcher(reader); ScoreDoc[] hits = indexSearcher.Search(booleanQuery, filter, 1000).ScoreDocs; Assert.AreEqual(1, hits.Length, "Number of matched documents"); reader.Dispose(); }
public void TestCustomMergeScheduler() { // we don't really need to execute anything, just to make sure the custom MS // compiles. But ensure that it can be used as well, e.g., no other hidden // dependencies or something. Therefore, don't use any random API ! Directory dir = new RAMDirectory(); IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null); conf.SetMergeScheduler(new ReportingMergeScheduler()); IndexWriter writer = new IndexWriter(dir, conf); writer.AddDocument(new Document()); writer.Commit(); // trigger flush writer.AddDocument(new Document()); writer.Commit(); // trigger flush writer.ForceMerge(1); writer.Dispose(); dir.Dispose(); }
public virtual void TestOpenIfChangedNoChangesButSegmentMerges() { // test openIfChanged() when the taxonomy hasn't really changed, but segments // were merged. The NRT reader will be reopened, and ParentArray used to assert // that the new reader contains more ordinals than were given from the old // TaxReader version Directory dir = NewDirectory(); // hold onto IW to forceMerge // note how we don't close it, since DTW will close it. var iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(new LogByteSizeMergePolicy())); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriterAnonymousInnerClassHelper3(this, dir, iw); // add a category so that the following DTR open will cause a flush and // a new segment will be created writer.AddCategory(new FacetLabel("a")); var reader = new DirectoryTaxonomyReader(writer); Assert.AreEqual(2, reader.Size); Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length); // merge all the segments so that NRT reader thinks there's a change iw.ForceMerge(1); // now calling openIfChanged should trip on the wrong assert in ParetArray's ctor var newtr = TaxonomyReader.OpenIfChanged(reader); Assert.NotNull(newtr); reader.Dispose(); reader = newtr; Assert.AreEqual(2, reader.Size); Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length); reader.Dispose(); writer.Dispose(); dir.Dispose(); }
public virtual void TestOpenIfChangedMergedSegment() { // test openIfChanged() when all index segments were merged - used to be // a bug in ParentArray, caught by testOpenIfChangedManySegments - only // this test is not random Directory dir = NewDirectory(); // hold onto IW to forceMerge // note how we don't close it, since DTW will close it. IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(new LogByteSizeMergePolicy())); var writer = new DirectoryTaxonomyWriterAnonymousInnerClassHelper2(this, dir, iw) as DirectoryTaxonomyWriter; var reader = new DirectoryTaxonomyReader(writer); Assert.AreEqual(1, reader.Size); Assert.AreEqual(1, reader.ParallelTaxonomyArrays.Parents().Length); // add category and call forceMerge -- this should flush IW and merge segments down to 1 // in ParentArray.initFromReader, this used to fail assuming there are no parents. writer.AddCategory(new FacetLabel("1")); iw.ForceMerge(1); // now calling openIfChanged should trip on the bug var newtr = TaxonomyReader.OpenIfChanged(reader); Assert.NotNull(newtr); reader.Dispose(); reader = newtr; Assert.AreEqual(2, reader.Size); Assert.AreEqual(2, reader.ParallelTaxonomyArrays.Parents().Length); reader.Dispose(); writer.Dispose(); dir.Dispose(); }
public virtual void TestSparseIndex() { Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); for (int d = -20; d <= 20; d++) { Document doc = new Document(); doc.Add(NewStringField("id", Convert.ToString(d), Field.Store.NO)); doc.Add(NewStringField("body", "body", Field.Store.NO)); writer.AddDocument(doc); } writer.ForceMerge(1); writer.DeleteDocuments(new Term("id", "0")); writer.Dispose(); IndexReader reader = DirectoryReader.Open(dir); IndexSearcher search = NewSearcher(reader); Assert.IsTrue(reader.HasDeletions); ScoreDoc[] result; Query q = new TermQuery(new Term("body", "body")); result = search.Search(q, FieldCacheRangeFilter.NewByteRange("id", (sbyte?)-20, (sbyte?)20, T, T), 100).ScoreDocs; Assert.AreEqual(40, result.Length, "find all"); result = search.Search(q, FieldCacheRangeFilter.NewByteRange("id", (sbyte?)0, (sbyte?)20, T, T), 100).ScoreDocs; Assert.AreEqual(20, result.Length, "find all"); result = search.Search(q, FieldCacheRangeFilter.NewByteRange("id", (sbyte?)-20, (sbyte?)0, T, T), 100).ScoreDocs; Assert.AreEqual(20, result.Length, "find all"); result = search.Search(q, FieldCacheRangeFilter.NewByteRange("id", (sbyte?)10, (sbyte?)20, T, T), 100).ScoreDocs; Assert.AreEqual(11, result.Length, "find all"); result = search.Search(q, FieldCacheRangeFilter.NewByteRange("id", (sbyte?)-20, (sbyte?)-10, T, T), 100).ScoreDocs; Assert.AreEqual(11, result.Length, "find all"); reader.Dispose(); dir.Dispose(); }
private static IndexReader MakeEmptyIndex(Random random, int numDocs) { Debug.Assert(numDocs > 0); Directory d = new MockDirectoryWrapper(random, new RAMDirectory()); IndexWriter w = new IndexWriter(d, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(random))); for (int i = 0; i < numDocs; i++) { w.AddDocument(new Document()); } w.ForceMerge(1); w.Commit(); w.Dispose(); DirectoryReader reader = DirectoryReader.Open(d); return new AllDeletedFilterReader(LuceneTestCase.GetOnlySegmentReader(reader)); }
public virtual void TestMultiValuedField() { Directory indexStore = NewDirectory(); IndexWriter writer = new IndexWriter(indexStore, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); for (int i = 0; i < 5; i++) { Document doc = new Document(); doc.Add(new StringField("string", "a" + i, Field.Store.NO)); doc.Add(new StringField("string", "b" + i, Field.Store.NO)); writer.AddDocument(doc); } writer.ForceMerge(1); // enforce one segment to have a higher unique term count in all cases writer.Dispose(); Sort sort = new Sort(new SortField("string", SortField.Type_e.STRING), SortField.FIELD_DOC); // this should not throw AIOOBE or RuntimeEx IndexReader reader = DirectoryReader.Open(indexStore); IndexSearcher searcher = NewSearcher(reader); searcher.Search(new MatchAllDocsQuery(), null, 500, sort); reader.Dispose(); indexStore.Dispose(); }
// Test using various international locales with accented characters (which // sort differently depending on locale) // // Copied (and slightly modified) from // Lucene.Net.Search.TestSort.testInternationalSort() // // TODO: this test is really fragile. there are already 3 different cases, // depending upon unicode version. public virtual void TestCollationKeySort(Analyzer usAnalyzer, Analyzer franceAnalyzer, Analyzer swedenAnalyzer, Analyzer denmarkAnalyzer, string usResult, string frResult, string svResult, string dkResult) { Directory indexStore = NewDirectory(); IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false))); // document data: // the tracer field is used to determine which document was hit string[][] sortData = new string[][] { new string[] { "A", "x", "p\u00EAche", "p\u00EAche", "p\u00EAche", "p\u00EAche" }, new string[] { "B", "y", "HAT", "HAT", "HAT", "HAT" }, new string[] { "C", "x", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9" }, new string[] { "D", "y", "HUT", "HUT", "HUT", "HUT" }, new string[] { "E", "x", "peach", "peach", "peach", "peach" }, new string[] { "F", "y", "H\u00C5T", "H\u00C5T", "H\u00C5T", "H\u00C5T" }, new string[] { "G", "x", "sin", "sin", "sin", "sin" }, new string[] { "H", "y", "H\u00D8T", "H\u00D8T", "H\u00D8T", "H\u00D8T" }, new string[] { "I", "x", "s\u00EDn", "s\u00EDn", "s\u00EDn", "s\u00EDn" }, new string[] { "J", "y", "HOT", "HOT", "HOT", "HOT" } }; FieldType customType = new FieldType(); customType.Stored = true; for (int i = 0; i < sortData.Length; ++i) { Document doc = new Document(); doc.Add(new Field("tracer", sortData[i][0], customType)); doc.Add(new TextField("contents", sortData[i][1], Field.Store.NO)); if (sortData[i][2] != null) { doc.Add(new TextField("US", usAnalyzer.TokenStream("US", new StringReader(sortData[i][2])))); } if (sortData[i][3] != null) { doc.Add(new TextField("France", franceAnalyzer.TokenStream("France", new StringReader(sortData[i][3])))); } if (sortData[i][4] != null) { doc.Add(new TextField("Sweden", swedenAnalyzer.TokenStream("Sweden", new StringReader(sortData[i][4])))); } if (sortData[i][5] != null) { doc.Add(new TextField("Denmark", denmarkAnalyzer.TokenStream("Denmark", new StringReader(sortData[i][5])))); } writer.AddDocument(doc); } writer.ForceMerge(1); writer.Dispose(); IndexReader reader = DirectoryReader.Open(indexStore); IndexSearcher searcher = new IndexSearcher(reader); Sort sort = new Sort(); Query queryX = new TermQuery(new Term("contents", "x")); Query queryY = new TermQuery(new Term("contents", "y")); sort.SetSort(new SortField("US", SortField.Type_e.STRING)); AssertMatches(searcher, queryY, sort, usResult); sort.SetSort(new SortField("France", SortField.Type_e.STRING)); AssertMatches(searcher, queryX, sort, frResult); sort.SetSort(new SortField("Sweden", SortField.Type_e.STRING)); AssertMatches(searcher, queryY, sort, svResult); sort.SetSort(new SortField("Denmark", SortField.Type_e.STRING)); AssertMatches(searcher, queryY, sort, dkResult); reader.Dispose(); indexStore.Dispose(); }
public virtual void TestRAMDirectorySize() { Directory dir = NewFSDirectory(IndexDir); MockDirectoryWrapper ramDir = new MockDirectoryWrapper(Random(), new RAMDirectory(dir, NewIOContext(Random()))); dir.Dispose(); IndexWriter writer = new IndexWriter(ramDir, (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetOpenMode(IndexWriterConfig.OpenMode_e.APPEND)); writer.ForceMerge(1); Assert.AreEqual(ramDir.SizeInBytes(), ramDir.RecomputedSizeInBytes); ThreadClass[] threads = new ThreadClass[NumThreads]; for (int i = 0; i < NumThreads; i++) { int num = i; threads[i] = new ThreadAnonymousInnerClassHelper(this, writer, num); } for (int i = 0; i < NumThreads; i++) { threads[i].Start(); } for (int i = 0; i < NumThreads; i++) { threads[i].Join(); } writer.ForceMerge(1); Assert.AreEqual(ramDir.SizeInBytes(), ramDir.RecomputedSizeInBytes); writer.Dispose(); }
public virtual void CreateRandomTerms(int nDocs, int nTerms, double power, Directory dir) { int[] freq = new int[nTerms]; Terms = new Term[nTerms]; for (int i = 0; i < nTerms; i++) { int f = (nTerms + 1) - i; // make first terms less frequent freq[i] = (int)Math.Ceiling(Math.Pow(f, power)); Terms[i] = new Term("f", char.ToString((char)('A' + i))); } IndexWriter iw = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.CREATE)); for (int i = 0; i < nDocs; i++) { Document d = new Document(); for (int j = 0; j < nTerms; j++) { if (Random().Next(freq[j]) == 0) { d.Add(NewStringField("f", Terms[j].Text(), Field.Store.NO)); //System.out.println(d); } } iw.AddDocument(d); } iw.ForceMerge(1); iw.Dispose(); }
public virtual void TestPerFieldCodec() { int NUM_DOCS = AtLeast(173); if (VERBOSE) { Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS); } using (BaseDirectoryWrapper dir = NewDirectory()) { dir.CheckIndexOnClose = false; // we use a custom codec provider using (IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(new CustomPerFieldCodec()).SetMergePolicy(NewLogMergePolicy(3)))) { Documents.Document doc = new Documents.Document(); // uses default codec: doc.Add(NewTextField("field1", "this field uses the standard codec as the test", Field.Store.NO)); // uses pulsing codec: Field field2 = NewTextField("field2", "this field uses the pulsing codec as the test", Field.Store.NO); doc.Add(field2); Field idField = NewStringField("id", "", Field.Store.NO); doc.Add(idField); for (int i = 0; i < NUM_DOCS; i++) { idField.StringValue = "" + i; w.AddDocument(doc); if ((i + 1) % 10 == 0) { w.Commit(); } } if (VERBOSE) { Console.WriteLine("TEST: now delete id=77"); } w.DeleteDocuments(new Term("id", "77")); using (IndexReader r = DirectoryReader.Open(w, true)) { Assert.AreEqual(NUM_DOCS - 1, r.NumDocs); IndexSearcher s = NewSearcher(r); Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits); Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits); } if (VERBOSE) { Console.WriteLine("\nTEST: now delete 2nd doc"); } w.DeleteDocuments(new Term("id", "44")); if (VERBOSE) { Console.WriteLine("\nTEST: now force merge"); } w.ForceMerge(1); if (VERBOSE) { Console.WriteLine("\nTEST: now open reader"); } using (IndexReader r = DirectoryReader.Open(w, true)) { Assert.AreEqual(NUM_DOCS - 2, r.MaxDoc); Assert.AreEqual(NUM_DOCS - 2, r.NumDocs); IndexSearcher s = NewSearcher(r); Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits); Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits); Assert.AreEqual(1, s.Search(new TermQuery(new Term("id", "76")), 1).TotalHits); Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "77")), 1).TotalHits); Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "44")), 1).TotalHits); if (VERBOSE) { Console.WriteLine("\nTEST: now close NRT reader"); } } } } }