SetMergePolicy() public method

Expert: set the merge policy used by this writer.
public SetMergePolicy ( MergePolicy mp ) : void
mp MergePolicy
return void
コード例 #1
0
        public virtual void  TestForceFlush()
        {
            Directory dir = new RAMDirectory();

            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);

            writer.SetMaxBufferedDocs(10);
            writer.SetMergeFactor(10);
            LogDocMergePolicy mp = new LogDocMergePolicy(writer);

            mp.SetMinMergeDocs(100);
            writer.SetMergePolicy(mp);

            for (int i = 0; i < 100; i++)
            {
                AddDoc(writer);
                writer.Close();

                writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
                writer.SetMaxBufferedDocs(10);
                writer.SetMergePolicy(mp);
                mp.SetMinMergeDocs(100);
                writer.SetMergeFactor(10);
                CheckInvariants(writer);
            }

            writer.Close();
        }
コード例 #2
0
        public virtual void  TestNoOverMerge()
        {
            Directory dir = new RAMDirectory();

            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);

            writer.SetMaxBufferedDocs(10);
            writer.SetMergeFactor(10);
            writer.SetMergePolicy(new LogDocMergePolicy(writer));

            bool noOverMerge = false;

            for (int i = 0; i < 100; i++)
            {
                AddDoc(writer);
                CheckInvariants(writer);
                if (writer.GetNumBufferedDocuments() + writer.GetSegmentCount() >= 18)
                {
                    noOverMerge = true;
                }
            }
            Assert.IsTrue(noOverMerge);

            writer.Close();
        }
コード例 #3
0
        private IndexWriter NewWriter(Directory dir, bool create)
        {
            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), create, IndexWriter.MaxFieldLength.UNLIMITED);

            writer.SetMergePolicy(new LogDocMergePolicy(writer));
            return(writer);
        }
コード例 #4
0
        private IndexWriter NewWriter(Directory dir, bool create)
        {
            IndexWriter writer = new IndexWriter(dir, true, new WhitespaceAnalyzer(), create);

            writer.SetMergePolicy(new LogDocMergePolicy(writer));
            return(writer);
        }
コード例 #5
0
        public virtual void  TestMergeFactorChange()
        {
            Directory dir = new RAMDirectory();

            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);

            writer.SetMaxBufferedDocs(10);
            writer.SetMergeFactor(100);
            writer.SetMergePolicy(new LogDocMergePolicy(writer));

            for (int i = 0; i < 250; i++)
            {
                AddDoc(writer);
                CheckInvariants(writer);
            }

            writer.SetMergeFactor(5);

            // merge policy only fixes segments on levels where merges
            // have been triggered, so check invariants after all adds
            for (int i = 0; i < 10; i++)
            {
                AddDoc(writer);
            }
            CheckInvariants(writer);

            writer.Close();
        }
コード例 #6
0
        public static void ApplyTo(IndexWriter writer)
        {
            writer.MergeFactor = MergeFactor;
            writer.MaxMergeDocs = MaxMergeDocs;

            var mergePolicy = new LogByteSizeMergePolicy(writer)
            {
                MaxMergeDocs = MaxMergeDocs,
                MergeFactor = MergeFactor,
                MinMergeMB = MinMergeMB,
                MaxMergeMB = MaxMergeMB
            };
            writer.SetMergePolicy(mergePolicy);
        }           
コード例 #7
0
        public virtual void  TestMaxBufferedDocsChange()
        {
            Directory dir = new RAMDirectory();

            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED, null);

            writer.SetMaxBufferedDocs(101);
            writer.MergeFactor = 101;
            writer.SetMergePolicy(new LogDocMergePolicy(writer));

            // leftmost* segment has 1 doc
            // rightmost* segment has 100 docs
            for (int i = 1; i <= 100; i++)
            {
                for (int j = 0; j < i; j++)
                {
                    AddDoc(writer);
                    CheckInvariants(writer);
                }
                writer.Close();

                writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED, null);
                writer.SetMaxBufferedDocs(101);
                writer.MergeFactor = 101;
                writer.SetMergePolicy(new LogDocMergePolicy(writer));
            }

            writer.SetMaxBufferedDocs(10);
            writer.MergeFactor = 10;

            // merge policy only fixes segments on levels where merges
            // have been triggered, so check invariants after all adds
            for (int i = 0; i < 100; i++)
            {
                AddDoc(writer);
            }
            CheckInvariants(writer);

            for (int i = 100; i < 1000; i++)
            {
                AddDoc(writer);
            }
            writer.Commit(null);
            ((ConcurrentMergeScheduler)writer.MergeScheduler).Sync();
            writer.Commit(null);
            CheckInvariants(writer);

            writer.Close();
        }
コード例 #8
0
        public virtual void  TestDeleteMerging()
        {
            RAMDirectory directory = new MockRAMDirectory();

            IndexWriter writer           = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED);
            ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();

            writer.SetMergeScheduler(cms);

            LogDocMergePolicy mp = new LogDocMergePolicy(writer);

            writer.SetMergePolicy(mp);

            // Force degenerate merging so we can get a mix of
            // merging of segments with and without deletes at the
            // start:
            mp.MinMergeDocs = 1000;

            Document doc     = new Document();
            Field    idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);

            doc.Add(idField);
            for (int i = 0; i < 10; i++)
            {
                for (int j = 0; j < 100; j++)
                {
                    idField.SetValue(System.Convert.ToString(i * 100 + j));
                    writer.AddDocument(doc);
                }

                int delID = i;
                while (delID < 100 * (1 + i))
                {
                    writer.DeleteDocuments(new Term("id", "" + delID));
                    delID += 10;
                }

                writer.Commit();
            }

            writer.Close();
            IndexReader reader = IndexReader.Open(directory, true);

            // Verify that we did not lose any deletes...
            Assert.AreEqual(450, reader.NumDocs());
            reader.Close();
            directory.Close();
        }
コード例 #9
0
 public virtual void  TestNormalCase()
 {
     Directory dir = new RAMDirectory();
     
     IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
     writer.SetMaxBufferedDocs(10);
     writer.MergeFactor = 10;
     writer.SetMergePolicy(new LogDocMergePolicy(writer));
     
     for (int i = 0; i < 100; i++)
     {
         AddDoc(writer);
         CheckInvariants(writer);
     }
     
     writer.Close();
 }
コード例 #10
0
        public virtual void  TestHangOnClose()
        {
            Directory   dir    = new MockRAMDirectory();
            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);

            writer.SetMergePolicy(new LogByteSizeMergePolicy(writer));
            writer.SetMaxBufferedDocs(5);
            writer.SetUseCompoundFile(false);
            writer.SetMergeFactor(100);

            Document doc = new Document();

            doc.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
            for (int i = 0; i < 60; i++)
            {
                writer.AddDocument(doc);
            }
            writer.SetMaxBufferedDocs(200);
            Document doc2 = new Document();

            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
            for (int i = 0; i < 10; i++)
            {
                writer.AddDocument(doc2);
            }
            writer.Close();

            Directory dir2 = new MockRAMDirectory();

            writer = new IndexWriter(dir2, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
            LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy(writer);

            lmp.SetMinMergeMB(0.0001);
            writer.SetMergePolicy(lmp);
            writer.SetMergeFactor(4);
            writer.SetUseCompoundFile(false);
            writer.SetMergeScheduler(new SerialMergeScheduler());
            writer.AddIndexesNoOptimize(new Directory[] { dir });
            writer.Close();
            dir.Close();
            dir2.Close();
        }
コード例 #11
0
        public virtual void  TestNormalCase()
        {
            Directory dir = new RAMDirectory();

            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);

            writer.SetMaxBufferedDocs(10);
            writer.MergeFactor = 10;
            writer.SetMergePolicy(new LogDocMergePolicy(writer));

            for (int i = 0; i < 100; i++)
            {
                AddDoc(writer);
                CheckInvariants(writer);
            }

            writer.Close();
        }
コード例 #12
0
        /// <summary> Delete a document by term and return the doc id
        ///
        /// </summary>
        /// <returns>
        ///
        /// public static int deleteDocument(Term term, IndexWriter writer) throws
        /// IOException { IndexReader reader = writer.getReader(); TermDocs td =
        /// reader.termDocs(term); int doc = -1; //if (td.next()) { // doc = td.doc();
        /// //} //writer.deleteDocuments(term); td.close(); return doc; }
        /// </returns>
        public static void  CreateIndex(Directory dir1, System.String indexName, bool multiSegment)
        {
            IndexWriter w = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);

            w.SetMergePolicy(new LogDocMergePolicy(w));
            for (int i = 0; i < 100; i++)
            {
                w.AddDocument(CreateDocument(i, indexName, 4));
                if (multiSegment && (i % 10) == 0)
                {
                }
            }
            if (!multiSegment)
            {
                w.Optimize();
            }
            w.Close();
        }
コード例 #13
0
        public virtual void  TestMergeDocCount0()
        {
            Directory dir = new RAMDirectory();

            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED, null);

            writer.SetMergePolicy(new LogDocMergePolicy(writer));
            writer.SetMaxBufferedDocs(10);
            writer.MergeFactor = 100;

            for (int i = 0; i < 250; i++)
            {
                AddDoc(writer);
                CheckInvariants(writer);
            }
            writer.Close();

            IndexReader reader = IndexReader.Open(dir, false, null);

            reader.DeleteDocuments(new Term("content", "aaa"), null);
            reader.Close();

            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED, null);
            writer.SetMergePolicy(new LogDocMergePolicy(writer));
            writer.SetMaxBufferedDocs(10);
            writer.MergeFactor = 5;

            // merge factor is changed, so check invariants after all adds
            for (int i = 0; i < 10; i++)
            {
                AddDoc(writer);
            }
            writer.Commit(null);
            ((ConcurrentMergeScheduler)writer.MergeScheduler).Sync();
            writer.Commit(null);
            CheckInvariants(writer);
            Assert.AreEqual(10, writer.MaxDoc());

            writer.Close();
        }
コード例 #14
0
 public virtual void  TestNoOverMerge()
 {
     Directory dir = new RAMDirectory();
     
     IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
     writer.SetMaxBufferedDocs(10);
     writer.MergeFactor = 10;
     writer.SetMergePolicy(new LogDocMergePolicy(writer));
     
     bool noOverMerge = false;
     for (int i = 0; i < 100; i++)
     {
         AddDoc(writer);
         CheckInvariants(writer);
         if (writer.GetNumBufferedDocuments() + writer.GetSegmentCount() >= 18)
         {
             noOverMerge = true;
         }
     }
     Assert.IsTrue(noOverMerge);
     
     writer.Close();
 }
コード例 #15
0
        public virtual void  TestMergeDocCount0()
        {
            Directory dir = new RAMDirectory();

            IndexWriter writer = new IndexWriter(dir, true, new WhitespaceAnalyzer(), true);

            writer.SetMergePolicy(new LogDocMergePolicy(writer));
            writer.SetMaxBufferedDocs(10);
            writer.SetMergeFactor(100);

            for (int i = 0; i < 250; i++)
            {
                AddDoc(writer);
                CheckInvariants(writer);
            }
            writer.Close();

            IndexReader reader = IndexReader.Open(dir);

            reader.DeleteDocuments(new Term("content", "aaa"));
            reader.Close();

            writer = new IndexWriter(dir, true, new WhitespaceAnalyzer(), false);
            writer.SetMergePolicy(new LogDocMergePolicy(writer));
            writer.SetMaxBufferedDocs(10);
            writer.SetMergeFactor(5);

            // merge factor is changed, so check invariants after all adds
            for (int i = 0; i < 10; i++)
            {
                AddDoc(writer);
            }
            CheckInvariants(writer);
            Assert.AreEqual(10, writer.DocCount());

            writer.Close();
        }
コード例 #16
0
		private IndexWriter NewWriter(Directory dir, bool create)
		{
			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), create, IndexWriter.MaxFieldLength.UNLIMITED);
			writer.SetMergePolicy(new LogDocMergePolicy(writer));
			return writer;
		}
コード例 #17
0
        public virtual void  TestMergeDocCount0()
        {
            Directory dir = new RAMDirectory();
            
            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
            writer.SetMergePolicy(new LogDocMergePolicy(writer));
            writer.SetMaxBufferedDocs(10);
            writer.MergeFactor = 100;
            
            for (int i = 0; i < 250; i++)
            {
                AddDoc(writer);
                CheckInvariants(writer);
            }
            writer.Close();

            IndexReader reader = IndexReader.Open(dir, false);
            reader.DeleteDocuments(new Term("content", "aaa"));
            reader.Close();
            
            writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED);
            writer.SetMergePolicy(new LogDocMergePolicy(writer));
            writer.SetMaxBufferedDocs(10);
            writer.MergeFactor = 5;
            
            // merge factor is changed, so check invariants after all adds
            for (int i = 0; i < 10; i++)
            {
                AddDoc(writer);
            }
            writer.Commit();
            ((ConcurrentMergeScheduler)writer.MergeScheduler).Sync();
            writer.Commit();
            CheckInvariants(writer);
            Assert.AreEqual(10, writer.MaxDoc());
            
            writer.Close();
        }
コード例 #18
0
        public void TestHangOnClose()
        {
            Directory dir = new MockRAMDirectory();
            IndexWriter writer = new IndexWriter(dir, false, new WhitespaceAnalyzer(), true);
            writer.SetMergePolicy(new LogByteSizeMergePolicy());
            writer.SetMaxBufferedDocs(5);
            writer.SetUseCompoundFile(false);
            writer.SetMergeFactor(100);

            Document doc = new Document();
            doc.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
            for (int i = 0; i < 60; i++)
                writer.AddDocument(doc);
            writer.SetMaxBufferedDocs(200);
            Document doc2 = new Document();
            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
            doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
            for (int i = 0; i < 10; i++)
                writer.AddDocument(doc2);
            writer.Close();

            Directory dir2 = new MockRAMDirectory();
            writer = new IndexWriter(dir2, false, new WhitespaceAnalyzer(), true);
            LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();
            lmp.SetMinMergeMB(0.0001);
            writer.SetMergePolicy(lmp);
            writer.SetMergeFactor(4);
            writer.SetUseCompoundFile(false);
            writer.SetMergeScheduler(new SerialMergeScheduler());
            writer.AddIndexesNoOptimize(new Directory[] { dir });
            writer.Close();

            dir.Close();
            dir2.Close();
        }
コード例 #19
0
 public virtual void  TestMergeFactorChange()
 {
     Directory dir = new RAMDirectory();
     
     IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
     writer.SetMaxBufferedDocs(10);
     writer.MergeFactor = 100;
     writer.SetMergePolicy(new LogDocMergePolicy(writer));
     
     for (int i = 0; i < 250; i++)
     {
         AddDoc(writer);
         CheckInvariants(writer);
     }
     
     writer.MergeFactor = 5;
     
     // merge policy only fixes segments on levels where merges
     // have been triggered, so check invariants after all adds
     for (int i = 0; i < 10; i++)
     {
         AddDoc(writer);
     }
     CheckInvariants(writer);
     
     writer.Close();
 }
コード例 #20
0
 public virtual void  TestMaxBufferedDocsChange()
 {
     Directory dir = new RAMDirectory();
     
     IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
     writer.SetMaxBufferedDocs(101);
     writer.MergeFactor = 101;
     writer.SetMergePolicy(new LogDocMergePolicy(writer));
     
     // leftmost* segment has 1 doc
     // rightmost* segment has 100 docs
     for (int i = 1; i <= 100; i++)
     {
         for (int j = 0; j < i; j++)
         {
             AddDoc(writer);
             CheckInvariants(writer);
         }
         writer.Close();
         
         writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED);
         writer.SetMaxBufferedDocs(101);
         writer.MergeFactor = 101;
         writer.SetMergePolicy(new LogDocMergePolicy(writer));
     }
     
     writer.SetMaxBufferedDocs(10);
     writer.MergeFactor = 10;
     
     // merge policy only fixes segments on levels where merges
     // have been triggered, so check invariants after all adds
     for (int i = 0; i < 100; i++)
     {
         AddDoc(writer);
     }
     CheckInvariants(writer);
     
     for (int i = 100; i < 1000; i++)
     {
         AddDoc(writer);
     }
     writer.Commit();
     ((ConcurrentMergeScheduler) writer.MergeScheduler).Sync();
     writer.Commit();
     CheckInvariants(writer);
     
     writer.Close();
 }
コード例 #21
0
ファイル: TestIndexWriter.cs プロジェクト: Rationalle/ravendb
		public virtual void  TestTermVectorCorruption()
		{
			
			Directory dir = new MockRAMDirectory();
			for (int iter = 0; iter < 4; iter++)
			{
				bool autoCommit = 1 == iter / 2;
				IndexWriter writer = new IndexWriter(dir, autoCommit, new StandardAnalyzer());
				writer.SetMaxBufferedDocs(2);
				writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
				writer.SetMergeScheduler(new SerialMergeScheduler());
				writer.SetMergePolicy(new LogDocMergePolicy(writer));
				
				Document document = new Document();
				
				Field storedField = new Field("stored", "stored", Field.Store.YES, Field.Index.NO);
				document.Add(storedField);
				writer.AddDocument(document);
				writer.AddDocument(document);
				
				document = new Document();
				document.Add(storedField);
				Field termVectorField = new Field("termVector", "termVector", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
				
				document.Add(termVectorField);
				writer.AddDocument(document);
				writer.Optimize();
				writer.Close();
				
				IndexReader reader = IndexReader.Open(dir);
				for (int i = 0; i < reader.NumDocs(); i++)
				{
					reader.Document(i);
					reader.GetTermFreqVectors(i);
				}
				reader.Close();
				
				writer = new IndexWriter(dir, autoCommit, new StandardAnalyzer());
				writer.SetMaxBufferedDocs(2);
				writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
				writer.SetMergeScheduler(new SerialMergeScheduler());
				writer.SetMergePolicy(new LogDocMergePolicy(writer));
				
				Directory[] indexDirs = new Directory[]{new MockRAMDirectory(dir)};
				writer.AddIndexes(indexDirs);
				writer.Close();
			}
			dir.Close();
		}
コード例 #22
0
		public virtual void  TestForceFlush()
		{
			Directory dir = new RAMDirectory();
			
			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
			writer.SetMaxBufferedDocs(10);
			writer.SetMergeFactor(10);
			LogDocMergePolicy mp = new LogDocMergePolicy(writer);
			mp.SetMinMergeDocs(100);
			writer.SetMergePolicy(mp);
			
			for (int i = 0; i < 100; i++)
			{
				AddDoc(writer);
				writer.Close();
				
				writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
				writer.SetMaxBufferedDocs(10);
				writer.SetMergePolicy(mp);
				mp.SetMinMergeDocs(100);
				writer.SetMergeFactor(10);
				CheckInvariants(writer);
			}
			
			writer.Close();
		}
コード例 #23
0
		public virtual void  TestMergeDocCount0()
		{
			Directory dir = new RAMDirectory();
			
			IndexWriter writer = new IndexWriter(dir, true, new WhitespaceAnalyzer(), true);
			writer.SetMergePolicy(new LogDocMergePolicy(writer));
			writer.SetMaxBufferedDocs(10);
			writer.SetMergeFactor(100);
			
			for (int i = 0; i < 250; i++)
			{
				AddDoc(writer);
				CheckInvariants(writer);
			}
			writer.Close();
			
			IndexReader reader = IndexReader.Open(dir);
			reader.DeleteDocuments(new Term("content", "aaa"));
			reader.Close();
			
			writer = new IndexWriter(dir, true, new WhitespaceAnalyzer(), false);
			writer.SetMergePolicy(new LogDocMergePolicy(writer));
			writer.SetMaxBufferedDocs(10);
			writer.SetMergeFactor(5);
			
			// merge factor is changed, so check invariants after all adds
			for (int i = 0; i < 10; i++)
			{
				AddDoc(writer);
			}
			CheckInvariants(writer);
			Assert.AreEqual(10, writer.DocCount());
			
			writer.Close();
		}
コード例 #24
0
		private static void  CreateIndex(Directory dir, bool multiSegment)
		{
			IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer());
			
			w.SetMergePolicy(new LogDocMergePolicy());
			
			for (int i = 0; i < 100; i++)
			{
				w.AddDocument(CreateDocument(i, 4));
				if (multiSegment && (i % 10) == 0)
				{
					w.Flush();
				}
			}
			
			if (!multiSegment)
			{
				w.Optimize();
			}
			
			w.Close();
			
			IndexReader r = IndexReader.Open(dir);
			if (multiSegment)
			{
				Assert.IsTrue(r is MultiSegmentReader);
			}
			else
			{
				Assert.IsTrue(r is SegmentReader);
			}
			r.Close();
		}
コード例 #25
0
		public virtual void  TestHangOnClose()
		{
			
			Directory dir = new MockRAMDirectory();
			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
			writer.SetMergePolicy(new LogByteSizeMergePolicy(writer));
			writer.SetMaxBufferedDocs(5);
			writer.UseCompoundFile = false;
			writer.MergeFactor = 100;
			
			Document doc = new Document();
			doc.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
			for (int i = 0; i < 60; i++)
				writer.AddDocument(doc);
			writer.SetMaxBufferedDocs(200);
			Document doc2 = new Document();
			doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
			doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
			doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
			doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
			for (int i = 0; i < 10; i++)
				writer.AddDocument(doc2);
			writer.Close();
			
			Directory dir2 = new MockRAMDirectory();
			writer = new IndexWriter(dir2, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
			LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy(writer);
			lmp.MinMergeMB = 0.0001;
			writer.SetMergePolicy(lmp);
			writer.MergeFactor = 4;
			writer.UseCompoundFile = false;
			writer.SetMergeScheduler(new SerialMergeScheduler());
			writer.AddIndexesNoOptimize(new Directory[]{dir});
			writer.Close();
			dir.Close();
			dir2.Close();
		}
コード例 #26
0
		public virtual void  TestDeleteMerging()
		{
			
			RAMDirectory directory = new MockRAMDirectory();
			
			IndexWriter writer = new IndexWriter(directory, true, ANALYZER, true);
			ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
			writer.SetMergeScheduler(cms);
			
			LogDocMergePolicy mp = new LogDocMergePolicy(writer);
			writer.SetMergePolicy(mp);
			
			// Force degenerate merging so we can get a mix of
			// merging of segments with and without deletes at the
			// start:
			mp.SetMinMergeDocs(1000);
			
			Document doc = new Document();
			Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
			doc.Add(idField);
			for (int i = 0; i < 10; i++)
			{
				for (int j = 0; j < 100; j++)
				{
					idField.SetValue(System.Convert.ToString(i * 100 + j));
					writer.AddDocument(doc);
				}
				
				int delID = i;
				while (delID < 100 * (1 + i))
				{
					writer.DeleteDocuments(new Term("id", "" + delID));
					delID += 10;
				}
				
				writer.Flush();
			}
			
			writer.Close();
			IndexReader reader = IndexReader.Open(directory);
			// Verify that we did not lose any deletes...
			Assert.AreEqual(450, reader.NumDocs());
			reader.Close();
			directory.Close();
		}
コード例 #27
0
		public virtual void  TestOptimizeMaxNumSegments2()
		{
			MockRAMDirectory dir = new MockRAMDirectory();
			
			Document doc = new Document();
			doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.TOKENIZED));
			
			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
			LogDocMergePolicy ldmp = new LogDocMergePolicy();
			ldmp.SetMinMergeDocs(1);
			writer.SetMergePolicy(ldmp);
			writer.SetMergeFactor(4);
			writer.SetMaxBufferedDocs(2);
			
			for (int iter = 0; iter < 10; iter++)
			{
				
				for (int i = 0; i < 19; i++)
					writer.AddDocument(doc);
				
				writer.Flush();
				
				SegmentInfos sis = new SegmentInfos();
				((ConcurrentMergeScheduler) writer.GetMergeScheduler()).Sync();
				sis.Read(dir);
				
				int segCount = sis.Count;
				
				writer.Optimize(7);
				
				sis = new SegmentInfos();
				((ConcurrentMergeScheduler) writer.GetMergeScheduler()).Sync();
				sis.Read(dir);
				int optSegCount = sis.Count;
				
				if (segCount < 7)
					Assert.AreEqual(segCount, optSegCount);
				else
					Assert.AreEqual(7, optSegCount);
			}
		}
コード例 #28
0
		public virtual void  TestOptimizeMaxNumSegments()
		{
			
			MockRAMDirectory dir = new MockRAMDirectory();
			
			Document doc = new Document();
			doc.Add(new Field("content", "aaa", Field.Store.YES, Field.Index.TOKENIZED));
			
			for (int numDocs = 38; numDocs < 500; numDocs += 38)
			{
				IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
				LogDocMergePolicy ldmp = new LogDocMergePolicy();
				ldmp.SetMinMergeDocs(1);
				writer.SetMergePolicy(ldmp);
				writer.SetMergeFactor(5);
				writer.SetMaxBufferedDocs(2);
				for (int j = 0; j < numDocs; j++)
					writer.AddDocument(doc);
				writer.Close();
				
				SegmentInfos sis = new SegmentInfos();
				sis.Read(dir);
				int segCount = sis.Count;
				
				writer = new IndexWriter(dir, new WhitespaceAnalyzer());
				writer.SetMergePolicy(ldmp);
				writer.SetMergeFactor(5);
				writer.Optimize(3);
				writer.Close();
				
				sis = new SegmentInfos();
				sis.Read(dir);
				int optSegCount = sis.Count;
				
				if (segCount < 3)
					Assert.AreEqual(segCount, optSegCount);
				else
					Assert.AreEqual(3, optSegCount);
			}
		}
コード例 #29
0
		public virtual void  TestTermVectorCorruption3()
		{
			Directory dir = new MockRAMDirectory();
			IndexWriter writer = new IndexWriter(dir, false, new StandardAnalyzer());
			writer.SetMaxBufferedDocs(2);
			writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
			writer.SetMergeScheduler(new SerialMergeScheduler());
			writer.SetMergePolicy(new LogDocMergePolicy());
			
			Document document = new Document();
			
			document = new Document();
			Field storedField = new Field("stored", "stored", Field.Store.YES, Field.Index.NO);
			document.Add(storedField);
			Field termVectorField = new Field("termVector", "termVector", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
			document.Add(termVectorField);
			for (int i = 0; i < 10; i++)
				writer.AddDocument(document);
			writer.Close();
			
			writer = new IndexWriter(dir, false, new StandardAnalyzer());
			writer.SetMaxBufferedDocs(2);
			writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
			writer.SetMergeScheduler(new SerialMergeScheduler());
			writer.SetMergePolicy(new LogDocMergePolicy());
			for (int i = 0; i < 6; i++)
				writer.AddDocument(document);
			
			writer.Optimize();
			writer.Close();
			
			IndexReader reader = IndexReader.Open(dir);
			for (int i = 0; i < 10; i++)
			{
				reader.GetTermFreqVectors(i);
				reader.Document(i);
			}
			reader.Close();
			dir.Close();
		}
コード例 #30
0
		private IndexWriter NewWriter(Directory dir, bool create)
		{
			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), create);
			writer.SetMergePolicy(new LogDocMergePolicy());
			return writer;
		}
コード例 #31
0
		public static void  CreateIndex(Directory dir, bool multiSegment)
		{
			IndexWriter.Unlock(dir);
			IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
			
			w.SetMergePolicy(new LogDocMergePolicy(w));
			
			for (int i = 0; i < 100; i++)
			{
				w.AddDocument(CreateDocument(i, 4));
				if (multiSegment && (i % 10) == 0)
				{
					w.Flush();
				}
			}
			
			if (!multiSegment)
			{
				w.Optimize();
			}
			
			w.Close();
			
			IndexReader r = IndexReader.Open(dir);
			if (multiSegment)
			{
				Assert.IsTrue(r.GetSequentialSubReaders().Length > 1);
			}
			else
			{
				Assert.IsTrue(r.GetSequentialSubReaders().Length == 1);
			}
			r.Close();
		}
コード例 #32
0
		public virtual void  TestTermVectorCorruption2()
		{
			Directory dir = new MockRAMDirectory();
			for (int iter = 0; iter < 4; iter++)
			{
				bool autoCommit = 1 == iter / 2;
				IndexWriter writer = new IndexWriter(dir, autoCommit, new StandardAnalyzer());
				writer.SetMaxBufferedDocs(2);
				writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
				writer.SetMergeScheduler(new SerialMergeScheduler());
				writer.SetMergePolicy(new LogDocMergePolicy());
				
				Document document = new Document();
				
				Field storedField = new Field("stored", "stored", Field.Store.YES, Field.Index.NO);
				document.Add(storedField);
				writer.AddDocument(document);
				writer.AddDocument(document);
				
				document = new Document();
				document.Add(storedField);
				Field termVectorField = new Field("termVector", "termVector", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
				document.Add(termVectorField);
				writer.AddDocument(document);
				writer.Optimize();
				writer.Close();
				
				IndexReader reader = IndexReader.Open(dir);
				Assert.IsTrue(reader.GetTermFreqVectors(0) == null);
				Assert.IsTrue(reader.GetTermFreqVectors(1) == null);
				Assert.IsTrue(reader.GetTermFreqVectors(2) != null);
				reader.Close();
			}
			dir.Close();
		}
コード例 #33
0
 /// <summary> Delete a document by term and return the doc id
 /// 
 /// </summary>
 /// <returns>
 /// 
 /// public static int deleteDocument(Term term, IndexWriter writer) throws
 /// IOException { IndexReader reader = writer.getReader(); TermDocs td =
 /// reader.termDocs(term); int doc = -1; //if (td.next()) { // doc = td.doc();
 /// //} //writer.deleteDocuments(term); td.close(); return doc; }
 /// </returns>
 public static void  CreateIndex(Directory dir1, System.String indexName, bool multiSegment)
 {
     IndexWriter w = new IndexWriter(dir1, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
     w.SetMergePolicy(new LogDocMergePolicy(w));
     for (int i = 0; i < 100; i++)
     {
         w.AddDocument(CreateDocument(i, indexName, 4));
         if (multiSegment && (i % 10) == 0)
         {
         }
     }
     if (!multiSegment)
     {
         w.Optimize();
     }
     w.Close();
 }