public virtual void TestRandomExceptionsThreads() { random = new Random((int)(DateTime.Now.Ticks & 0x7fffffff)); MockRAMDirectory dir = new MockRAMDirectory(); MockIndexWriter writer = new MockIndexWriter(this, dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); ((ConcurrentMergeScheduler)writer.MergeScheduler).SetSuppressExceptions(); //writer.setMaxBufferedDocs(10); writer.SetRAMBufferSizeMB(0.2); if (DEBUG) { System.IO.StreamWriter temp_writer; temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding); temp_writer.AutoFlush = true; writer.SetInfoStream(temp_writer, null); } int NUM_THREADS = 4; IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(this, i, writer); threads[i].Start(); } for (int i = 0; i < NUM_THREADS; i++) { threads[i].Join(); } for (int i = 0; i < NUM_THREADS; i++) { Assert.IsNull(threads[i].failure, "thread " + threads[i].Name + ": hit unexpected failure"); } writer.Commit(null); try { writer.Close(); } catch (System.Exception t) { System.Console.Out.WriteLine("exception during close:"); System.Console.Out.WriteLine(t.StackTrace); writer.Rollback(null); } // Confirm that when doc hits exception partway through tokenization, it's deleted: IndexReader r2 = IndexReader.Open((Directory)dir, true, null); int count = r2.DocFreq(new Term("content4", "aaa"), null); int count2 = r2.DocFreq(new Term("content4", "ddd"), null); Assert.AreEqual(count, count2); r2.Close(); _TestUtil.CheckIndex(dir); }
public virtual DocsAndWriter IndexRandomIWReader(int nThreads, int iterations, int range, Directory dir) { System.Collections.Hashtable docs = new System.Collections.Hashtable(); IndexWriter w = new MockIndexWriter(this, dir, autoCommit, new WhitespaceAnalyzer(), true); w.SetUseCompoundFile(false); /*** * w.setMaxMergeDocs(Integer.MAX_VALUE); * w.setMaxFieldLength(10000); * w.setRAMBufferSizeMB(1); * w.setMergeFactor(10); ***/ // force many merges w.SetMergeFactor(mergeFactor); w.SetRAMBufferSizeMB(.1); w.SetMaxBufferedDocs(maxBufferedDocs); threads = new IndexingThread[nThreads]; for (int i = 0; i < threads.Length; i++) { IndexingThread th = new IndexingThread(); th.w = w; th.base_Renamed = 1000000 * i; th.range = range; th.iterations = iterations; threads[i] = th; } for (int i = 0; i < threads.Length; i++) { threads[i].Start(); } for (int i = 0; i < threads.Length; i++) { threads[i].Join(); } // w.optimize(); //w.close(); for (int i = 0; i < threads.Length; i++) { IndexingThread th = threads[i]; lock (th) { SupportClass.CollectionsHelper.AddAllIfNotContains(docs, th.docs); } } _TestUtil.CheckIndex(dir); DocsAndWriter dw = new DocsAndWriter(); dw.docs = docs; dw.writer = w; return(dw); }
public virtual System.Collections.IDictionary IndexRandom(int nThreads, int iterations, int range, Directory dir) { System.Collections.IDictionary docs = new System.Collections.Hashtable(); for (int iter = 0; iter < 3; iter++) { IndexWriter w = new MockIndexWriter(this, dir, autoCommit, new WhitespaceAnalyzer(), true); w.SetUseCompoundFile(false); // force many merges w.SetMergeFactor(mergeFactor); w.SetRAMBufferSizeMB(.1); w.SetMaxBufferedDocs(maxBufferedDocs); threads = new IndexingThread[nThreads]; for (int i = 0; i < threads.Length; i++) { IndexingThread th = new IndexingThread(); th.w = w; th.base_Renamed = 1000000 * i; th.range = range; th.iterations = iterations; threads[i] = th; } for (int i = 0; i < threads.Length; i++) { threads[i].Start(); } for (int i = 0; i < threads.Length; i++) { threads[i].Join(); } // w.optimize(); w.Close(); for (int i = 0; i < threads.Length; i++) { IndexingThread th = threads[i]; lock (th) { System.Collections.IEnumerator e = th.docs.Keys.GetEnumerator(); while (e.MoveNext()) { docs[e.Current] = th.docs[e.Current]; } } } } _TestUtil.CheckIndex(dir); return(docs); }
public virtual void TestRandomExceptions() { MockRAMDirectory dir = new MockRAMDirectory(); MockIndexWriter writer = new MockIndexWriter(this, dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); ((ConcurrentMergeScheduler)writer.GetMergeScheduler()).SetSuppressExceptions(); //writer.setMaxBufferedDocs(10); writer.SetRAMBufferSizeMB(0.1); if (DEBUG) { System.IO.StreamWriter temp_writer; temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding); temp_writer.AutoFlush = true; writer.SetInfoStream(temp_writer); } IndexerThread thread = new IndexerThread(this, 0, writer); thread.Run(); if (thread.failure != null) { System.Console.Out.WriteLine(thread.failure.StackTrace); Assert.Fail("thread " + thread.Name + ": hit unexpected failure"); } writer.Commit(); try { writer.Close(); } catch (System.Exception t) { System.Console.Out.WriteLine("exception during close:"); System.Console.Out.WriteLine(t.StackTrace); writer.Rollback(); } // Confirm that when doc hits exception partway through tokenization, it's deleted: IndexReader r2 = IndexReader.Open(dir); int count = r2.DocFreq(new Term("content4", "aaa")); int count2 = r2.DocFreq(new Term("content4", "ddd")); Assert.AreEqual(count, count2); r2.Close(); _TestUtil.CheckIndex(dir); }
public virtual System.Collections.IDictionary IndexRandom(int nThreads, int iterations, int range, Directory dir) { System.Collections.IDictionary docs = new System.Collections.Hashtable(); for (int iter = 0; iter < 3; iter++) { IndexWriter w = new MockIndexWriter(this, dir, autoCommit, new WhitespaceAnalyzer(), true); w.SetUseCompoundFile(false); // force many merges w.SetMergeFactor(mergeFactor); w.SetRAMBufferSizeMB(.1); w.SetMaxBufferedDocs(maxBufferedDocs); threads = new IndexingThread[nThreads]; for (int i = 0; i < threads.Length; i++) { IndexingThread th = new IndexingThread(); th.w = w; th.base_Renamed = 1000000 * i; th.range = range; th.iterations = iterations; threads[i] = th; } for (int i = 0; i < threads.Length; i++) { threads[i].Start(); } for (int i = 0; i < threads.Length; i++) { threads[i].Join(); } // w.optimize(); w.Close(); for (int i = 0; i < threads.Length; i++) { IndexingThread th = threads[i]; lock (th) { System.Collections.IEnumerator e = th.docs.Keys.GetEnumerator(); while (e.MoveNext()) { docs[e.Current] = th.docs[e.Current]; } } } } _TestUtil.CheckIndex(dir); return docs; }
public virtual DocsAndWriter IndexRandomIWReader(int nThreads, int iterations, int range, Directory dir) { System.Collections.Hashtable docs = new System.Collections.Hashtable(); IndexWriter w = new MockIndexWriter(this, dir, autoCommit, new WhitespaceAnalyzer(), true); w.SetUseCompoundFile(false); /*** w.setMaxMergeDocs(Integer.MAX_VALUE); w.setMaxFieldLength(10000); w.setRAMBufferSizeMB(1); w.setMergeFactor(10); ***/ // force many merges w.SetMergeFactor(mergeFactor); w.SetRAMBufferSizeMB(.1); w.SetMaxBufferedDocs(maxBufferedDocs); threads = new IndexingThread[nThreads]; for (int i = 0; i < threads.Length; i++) { IndexingThread th = new IndexingThread(); th.w = w; th.base_Renamed = 1000000 * i; th.range = range; th.iterations = iterations; threads[i] = th; } for (int i = 0; i < threads.Length; i++) { threads[i].Start(); } for (int i = 0; i < threads.Length; i++) { threads[i].Join(); } // w.optimize(); //w.close(); for (int i = 0; i < threads.Length; i++) { IndexingThread th = threads[i]; lock (th) { SupportClass.CollectionsHelper.AddAllIfNotContains(docs, th.docs); } } _TestUtil.CheckIndex(dir); DocsAndWriter dw = new DocsAndWriter(); dw.docs = docs; dw.writer = w; return dw; }
public virtual void TestRandomExceptionsThreads() { MockRAMDirectory dir = new MockRAMDirectory(); MockIndexWriter writer = new MockIndexWriter(this, dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); ((ConcurrentMergeScheduler) writer.GetMergeScheduler()).SetSuppressExceptions(); //writer.setMaxBufferedDocs(10); writer.SetRAMBufferSizeMB(0.2); if (DEBUG) { System.IO.StreamWriter temp_writer; temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding); temp_writer.AutoFlush = true; writer.SetInfoStream(temp_writer); } int NUM_THREADS = 4; IndexerThread[] threads = new IndexerThread[NUM_THREADS]; for (int i = 0; i < NUM_THREADS; i++) { threads[i] = new IndexerThread(this, i, writer); threads[i].Start(); } for (int i = 0; i < NUM_THREADS; i++) threads[i].Join(); for (int i = 0; i < NUM_THREADS; i++) if (threads[i].failure != null) Assert.Fail("thread " + threads[i].Name + ": hit unexpected failure"); writer.Commit(); try { writer.Close(); } catch (System.Exception t) { System.Console.Out.WriteLine("exception during close:"); System.Console.Out.WriteLine(t.StackTrace); writer.Rollback(); } // Confirm that when doc hits exception partway through tokenization, it's deleted: IndexReader r2 = IndexReader.Open(dir); int count = r2.DocFreq(new Term("content4", "aaa")); int count2 = r2.DocFreq(new Term("content4", "ddd")); Assert.AreEqual(count, count2); r2.Close(); _TestUtil.CheckIndex(dir); }