public ThreadAnonymousInnerClassHelper(TestAddTaxonomy outerInstance, int range, AtomicInteger numCats, DirectoryTaxonomyWriter tw) { this.outerInstance = outerInstance; this.range = range; this.numCats = numCats; this.tw = tw; }
private void Dotest(int ncats, int range) { AtomicInteger numCats = new AtomicInteger(ncats); Directory[] dirs = new Directory[2]; for (int i = 0; i < dirs.Length; i++) { dirs[i] = NewDirectory(); var tw = new DirectoryTaxonomyWriter(dirs[i]); ThreadClass[] addThreads = new ThreadClass[4]; for (int j = 0; j < addThreads.Length; j++) { addThreads[j] = new ThreadAnonymousInnerClassHelper(this, range, numCats, tw); } foreach (ThreadClass t in addThreads) { t.Start(); } foreach (ThreadClass t in addThreads) { t.Join(); } tw.Dispose(); } var tw1 = new DirectoryTaxonomyWriter(dirs[0]); IOrdinalMap map = randomOrdinalMap(); tw1.AddTaxonomy(dirs[1], map); tw1.Dispose(); validate(dirs[0], dirs[1], map); IOUtils.Close(dirs); }
protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled) { int numDocumentsToIndex = 10 + AtLeast(30); AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); Directory dir = NewDirectory(); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); MockAnalyzer analyzer = new MockAnalyzer(Random()); analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy); int numDWPT = 1 + AtLeast(2); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT); iwc.SetIndexerThreadPool(threadPool); iwc.SetRAMBufferSizeMB(maxRamMB); iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy; Assert.IsFalse(flushPolicy.FlushOnDocCount()); Assert.IsFalse(flushPolicy.FlushOnDeleteTerms()); Assert.IsTrue(flushPolicy.FlushOnRAM()); DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.FlushControl; Assert.AreEqual(0, flushControl.FlushBytes(), " bytes must be 0 after init"); IndexThread[] threads = new IndexThread[numThreads]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, false); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0); Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due numThreads=" + numThreads); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs()); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark"); AssertActiveBytesAfter(flushControl); if (flushPolicy.HasMarkedPending) { Assert.IsTrue(maxRAMBytes < flushControl.PeakActiveBytes); } if (ensureNotStalled) { Assert.IsFalse(docsWriter.FlushControl.StallControl.WasStalled()); } writer.Dispose(); Assert.AreEqual(0, flushControl.ActiveBytes()); dir.Dispose(); }
public virtual void TestApplyDeletesOnFlush() { Directory dir = NewDirectory(); // Cannot use RandomIndexWriter because we don't want to // ever call commit() for this test: AtomicInteger docsInSegment = new AtomicInteger(); AtomicBoolean closing = new AtomicBoolean(); AtomicBoolean sawAfterFlush = new AtomicBoolean(); IndexWriter w = new IndexWriterAnonymousInnerClassHelper(this, dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetRAMBufferSizeMB(0.5).SetMaxBufferedDocs(-1).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).SetReaderPooling(false), docsInSegment, closing, sawAfterFlush); int id = 0; while (true) { StringBuilder sb = new StringBuilder(); for (int termIDX = 0; termIDX < 100; termIDX++) { sb.Append(' ').Append(TestUtil.RandomRealisticUnicodeString(Random())); } if (id == 500) { w.DeleteDocuments(new Term("id", "0")); } Document doc = new Document(); doc.Add(NewStringField("id", "" + id, Field.Store.NO)); doc.Add(NewTextField("body", sb.ToString(), Field.Store.NO)); w.UpdateDocument(new Term("id", "" + id), doc); docsInSegment.IncrementAndGet(); // TODO: fix this test if (SlowFileExists(dir, "_0_1.del") || SlowFileExists(dir, "_0_1.liv")) { if (VERBOSE) { Console.WriteLine("TEST: deletes created @ id=" + id); } break; } id++; } closing.Set(true); Assert.IsTrue(sawAfterFlush.Get()); w.Dispose(); dir.Dispose(); }
public virtual void TestBooleanScorerMax() { Directory dir = NewDirectory(); RandomIndexWriter riw = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); int docCount = AtLeast(10000); for (int i = 0; i < docCount; i++) { Document doc = new Document(); doc.Add(NewField("field", "a", TextField.TYPE_NOT_STORED)); riw.AddDocument(doc); } riw.ForceMerge(1); IndexReader r = riw.Reader; riw.Dispose(); IndexSearcher s = NewSearcher(r); BooleanQuery bq = new BooleanQuery(); bq.Add(new TermQuery(new Term("field", "a")), BooleanClause.Occur.SHOULD); bq.Add(new TermQuery(new Term("field", "a")), BooleanClause.Occur.SHOULD); Weight w = s.CreateNormalizedWeight(bq); Assert.AreEqual(1, s.IndexReader.Leaves.Count); BulkScorer scorer = w.BulkScorer(s.IndexReader.Leaves[0], false, null); FixedBitSet hits = new FixedBitSet(docCount); AtomicInteger end = new AtomicInteger(); Collector c = new CollectorAnonymousInnerClassHelper(this, scorer, hits, end); while (end.Get() < docCount) { int inc = TestUtil.NextInt(Random(), 1, 1000); end.AddAndGet(inc); scorer.Score(c, end.Get()); } Assert.AreEqual(docCount, hits.Cardinality()); r.Dispose(); dir.Dispose(); }
public virtual void TestConcurrency() { int ncats = AtLeast(100000); // add many categories int range = ncats * 3; // affects the categories selection AtomicInteger numCats = new AtomicInteger(ncats); Directory dir = NewDirectory(); var values = new ConcurrentDictionary<string, string>(); double d = Random().NextDouble(); ITaxonomyWriterCache cache; if (d < 0.7) { // this is the fastest, yet most memory consuming cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3); } else if (TEST_NIGHTLY && d > 0.98) { // this is the slowest, but tests the writer concurrency when no caching is done. // only pick it during NIGHTLY tests, and even then, with very low chances. cache = NO_OP_CACHE; } else { // this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too. cache = new LruTaxonomyWriterCache(ncats / 10); } if (VERBOSE) { Console.WriteLine("TEST: use cache=" + cache); } var tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache); ThreadClass[] addThreads = new ThreadClass[AtLeast(4)]; for (int z = 0; z < addThreads.Length; z++) { addThreads[z] = new ThreadAnonymousInnerClassHelper(this, range, numCats, values, tw); } foreach (var t in addThreads) { t.Start(); } foreach (var t in addThreads) { t.Join(); } tw.Dispose(); DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir); // +1 for root category if (values.Count + 1 != dtr.Count) { foreach (string value in values.Keys) { FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value)); if (dtr.GetOrdinal(label) == -1) { Console.WriteLine("FAIL: path=" + label + " not recognized"); } } Fail("mismatch number of categories"); } int[] parents = dtr.ParallelTaxonomyArrays.Parents; foreach (string cat in values.Keys) { FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat)); Assert.True(dtr.GetOrdinal(cp) > 0, "category not found " + cp); int level = cp.Length; int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0) FacetLabel path = new FacetLabel(); for (int i = 0; i < level; i++) { path = cp.Subpath(i + 1); int ord = dtr.GetOrdinal(path); Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path); parentOrd = ord; // next level should have this parent } } IOUtils.Close(dtr, dir); }
public virtual void TestMaxMergeCount() { Directory dir = NewDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); int maxMergeCount = TestUtil.NextInt(Random(), 1, 5); int maxMergeThreads = TestUtil.NextInt(Random(), 1, maxMergeCount); CountdownEvent enoughMergesWaiting = new CountdownEvent(maxMergeCount); AtomicInteger runningMergeCount = new AtomicInteger(0); AtomicBoolean failed = new AtomicBoolean(); if (VERBOSE) { Console.WriteLine("TEST: maxMergeCount=" + maxMergeCount + " maxMergeThreads=" + maxMergeThreads); } ConcurrentMergeScheduler cms = new ConcurrentMergeSchedulerAnonymousInnerClassHelper(this, maxMergeCount, enoughMergesWaiting, runningMergeCount, failed); cms.SetMaxMergesAndThreads(maxMergeCount, maxMergeThreads); iwc.SetMergeScheduler(cms); iwc.SetMaxBufferedDocs(2); TieredMergePolicy tmp = new TieredMergePolicy(); iwc.SetMergePolicy(tmp); tmp.MaxMergeAtOnce = 2; tmp.SegmentsPerTier = 2; IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); doc.Add(NewField("field", "field", TextField.TYPE_NOT_STORED)); while (enoughMergesWaiting.CurrentCount != 0 && !failed.Get()) { for (int i = 0; i < 10; i++) { w.AddDocument(doc); } } w.Dispose(false); dir.Dispose(); }
public CountListener(AtomicInteger count) { this.Count = count; }
public ThreadAnonymousInnerClassHelper(TestNumericDocValuesUpdates outerInstance, string str, IndexWriter writer, int numDocs, CountDownLatch done, AtomicInteger numUpdates, string f, string cf) : base(str) { this.OuterInstance = outerInstance; this.Writer = writer; this.NumDocs = numDocs; this.Done = done; this.NumUpdates = numUpdates; this.f = f; this.Cf = cf; }
public ThreadAnonymousInnerClassHelper(TestConcurrentFacetedIndexing outerInstance, AtomicInteger numDocs, ConcurrentDictionary<string, string> values, IndexWriter iw, Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter tw, FacetsConfig config) { this.outerInstance = outerInstance; this.numDocs = numDocs; this.values = values; this.iw = iw; this.tw = tw; this.config = config; }
public IndexWriterAnonymousInnerClassHelper(TestIndexWriterDelete outerInstance, Directory dir, IndexWriterConfig setReaderPooling, AtomicInteger docsInSegment, AtomicBoolean closing, AtomicBoolean sawAfterFlush) : base(dir, setReaderPooling) { this.OuterInstance = outerInstance; this.DocsInSegment = docsInSegment; this.Closing = closing; this.SawAfterFlush = sawAfterFlush; }
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInteger index, int?[] ids, CountDownLatch latch) { this.Queue = queue; this.Index = index; this.Ids = ids; this.Slice = queue.NewSlice(); Deletes = new BufferedUpdates(); this.Latch = latch; }
public virtual void TestConcurrency() { AtomicInteger numDocs = new AtomicInteger(AtLeast(10000)); Directory indexDir = NewDirectory(); Directory taxoDir = NewDirectory(); ConcurrentDictionary<string, string> values = new ConcurrentDictionary<string, string>(); IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, null)); var tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, NewTaxoWriterCache(numDocs.Get())); ThreadClass[] indexThreads = new ThreadClass[AtLeast(4)]; FacetsConfig config = new FacetsConfig(); for (int i = 0; i < 10; i++) { config.SetHierarchical("l1." + i, true); config.SetMultiValued("l1." + i, true); } for (int i = 0; i < indexThreads.Length; i++) { indexThreads[i] = new ThreadAnonymousInnerClassHelper(this, numDocs, values, iw, tw, config); } foreach (ThreadClass t in indexThreads) { t.Start(); } foreach (ThreadClass t in indexThreads) { t.Join(); } var tr = new DirectoryTaxonomyReader(tw); // +1 for root category if (values.Count + 1 != tr.Count) { foreach (string value in values.Keys) { FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value)); if (tr.GetOrdinal(label) == -1) { Console.WriteLine("FAIL: path=" + label + " not recognized"); } } Fail("mismatch number of categories"); } int[] parents = tr.ParallelTaxonomyArrays.Parents; foreach (string cat in values.Keys) { FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat)); Assert.True(tr.GetOrdinal(cp) > 0, "category not found " + cp); int level = cp.Length; int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0) FacetLabel path = null; for (int i = 0; i < level; i++) { path = cp.Subpath(i + 1); int ord = tr.GetOrdinal(path); Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path); parentOrd = ord; // next level should have this parent } } IOUtils.Close(tw, iw, tr, taxoDir, indexDir); }
public virtual void TestStressDeleteQueue() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); HashSet<Term> uniqueValues = new HashSet<Term>(); int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random().Next(); uniqueValues.Add(new Term("id", ids[i].ToString())); } CountDownLatch latch = new CountDownLatch(1); AtomicInteger index = new AtomicInteger(0); int numThreads = 2 + Random().Next(5); UpdateThread[] threads = new UpdateThread[numThreads]; for (int i = 0; i < threads.Length; i++) { threads[i] = new UpdateThread(queue, index, ids, latch); threads[i].Start(); } latch.countDown(); for (int i = 0; i < threads.Length; i++) { threads[i].Join(); } foreach (UpdateThread updateThread in threads) { DeleteSlice slice = updateThread.Slice; queue.UpdateSlice(slice); BufferedUpdates deletes = updateThread.Deletes; slice.Apply(deletes, BufferedUpdates.MAX_INT); Assert.AreEqual(uniqueValues, deletes.Terms_Nunit().Keys); } queue.TryApplyGlobalSlice(); HashSet<Term> frozenSet = new HashSet<Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes()); frozenSet.Add(new Term(t.Field(), bytesRef)); } Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze"); Assert.AreEqual(uniqueValues.Count, frozenSet.Count); Assert.AreEqual(uniqueValues, frozenSet); }
public virtual void Test() { // update variables int commitPercent = Random().Next(20); int softCommitPercent = Random().Next(100); // what percent of the commits are soft int deletePercent = Random().Next(50); int deleteByQueryPercent = Random().Next(25); int ndocs = AtLeast(50); int nWriteThreads = TestUtil.NextInt(Random(), 1, TEST_NIGHTLY ? 10 : 5); int maxConcurrentCommits = TestUtil.NextInt(Random(), 1, TEST_NIGHTLY ? 10 : 5); // number of committers at a time... needed if we want to avoid commit errors due to exceeding the max bool tombstones = Random().NextBoolean(); // query variables AtomicLong operations = new AtomicLong(AtLeast(10000)); // number of query operations to perform in total int nReadThreads = TestUtil.NextInt(Random(), 1, TEST_NIGHTLY ? 10 : 5); InitModel(ndocs); FieldType storedOnlyType = new FieldType(); storedOnlyType.Stored = true; if (VERBOSE) { Console.WriteLine("\n"); Console.WriteLine("TEST: commitPercent=" + commitPercent); Console.WriteLine("TEST: softCommitPercent=" + softCommitPercent); Console.WriteLine("TEST: deletePercent=" + deletePercent); Console.WriteLine("TEST: deleteByQueryPercent=" + deleteByQueryPercent); Console.WriteLine("TEST: ndocs=" + ndocs); Console.WriteLine("TEST: nWriteThreads=" + nWriteThreads); Console.WriteLine("TEST: nReadThreads=" + nReadThreads); Console.WriteLine("TEST: maxConcurrentCommits=" + maxConcurrentCommits); Console.WriteLine("TEST: tombstones=" + tombstones); Console.WriteLine("TEST: operations=" + operations); Console.WriteLine("\n"); } AtomicInteger numCommitting = new AtomicInteger(); IList<ThreadClass> threads = new List<ThreadClass>(); Directory dir = NewDirectory(); RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); writer.DoRandomForceMergeAssert = false; writer.Commit(); Reader = DirectoryReader.Open(dir); for (int i = 0; i < nWriteThreads; i++) { ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, "WRITER" + i, commitPercent, softCommitPercent, deletePercent, deleteByQueryPercent, ndocs, maxConcurrentCommits, tombstones, operations, storedOnlyType, numCommitting, writer); threads.Add(thread); } for (int i = 0; i < nReadThreads; i++) { ThreadClass thread = new ThreadAnonymousInnerClassHelper2(this, "READER" + i, ndocs, tombstones, operations); threads.Add(thread); } foreach (ThreadClass thread in threads) { thread.Start(); } foreach (ThreadClass thread in threads) { thread.Join(); } writer.Dispose(); if (VERBOSE) { Console.WriteLine("TEST: close reader=" + Reader); } Reader.Dispose(); dir.Dispose(); }
public ThreadAnonymousInnerClassHelper(TestStressNRT outerInstance, string str, int commitPercent, int softCommitPercent, int deletePercent, int deleteByQueryPercent, int ndocs, int maxConcurrentCommits, bool tombstones, AtomicLong operations, FieldType storedOnlyType, AtomicInteger numCommitting, RandomIndexWriter writer) : base(str) { this.OuterInstance = outerInstance; this.CommitPercent = commitPercent; this.SoftCommitPercent = softCommitPercent; this.DeletePercent = deletePercent; this.DeleteByQueryPercent = deleteByQueryPercent; this.Ndocs = ndocs; this.MaxConcurrentCommits = maxConcurrentCommits; this.Tombstones = tombstones; this.Operations = operations; this.StoredOnlyType = storedOnlyType; this.NumCommitting = numCommitting; this.Writer = writer; rand = new Random(Random().Next()); }
protected internal virtual void RunSearchThreads(DateTime stopTime) { int numThreads = TestUtil.NextInt(Random(), 1, 5); ThreadClass[] searchThreads = new ThreadClass[numThreads]; AtomicInteger totHits = new AtomicInteger(); // silly starting guess: AtomicInteger totTermCount = new AtomicInteger(100); // TODO: we should enrich this to do more interesting searches for (int thread = 0; thread < searchThreads.Length; thread++) { searchThreads[thread] = new ThreadAnonymousInnerClassHelper2(this, stopTime, totHits, totTermCount); searchThreads[thread].SetDaemon(true); searchThreads[thread].Start(); } for (int thread = 0; thread < searchThreads.Length; thread++) { searchThreads[thread].Join(); } if (VERBOSE) { Console.WriteLine("TEST: DONE search: totHits=" + totHits); } }
public virtual void TestFlushDocCount() { int[] numThreads = new int[] { 2 + AtLeast(1), 1 }; for (int i = 0; i < numThreads.Length; i++) { int numDocumentsToIndex = 50 + AtLeast(30); AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); Directory dir = NewDirectory(); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetFlushPolicy(flushPolicy); int numDWPT = 1 + AtLeast(2); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT); iwc.SetIndexerThreadPool(threadPool); iwc.SetMaxBufferedDocs(2 + AtLeast(10)); iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy; Assert.IsTrue(flushPolicy.FlushOnDocCount()); Assert.IsFalse(flushPolicy.FlushOnDeleteTerms()); Assert.IsFalse(flushPolicy.FlushOnRAM()); DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.FlushControl; Assert.AreEqual(0, flushControl.FlushBytes(), " bytes must be 0 after init"); IndexThread[] threads = new IndexThread[numThreads[i]]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(this, numDocs, numThreads[i], writer, LineDocFile, false); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due numThreads=" + numThreads[i]); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs()); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); Assert.IsTrue(flushPolicy.PeakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark"); AssertActiveBytesAfter(flushControl); writer.Dispose(); Assert.AreEqual(0, flushControl.ActiveBytes()); dir.Dispose(); } }
public ThreadAnonymousInnerClassHelper2(ThreadedIndexingAndSearchingTestCase outerInstance, DateTime stopTime, AtomicInteger totHits, AtomicInteger totTermCount) { this.OuterInstance = outerInstance; this.StopTime = stopTime; this.TotHits = totHits; this.TotTermCount = totTermCount; }
public virtual void TestRandom() { int numThreads = 1 + Random().Next(8); int numDocumentsToIndex = 50 + AtLeast(70); AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); Directory dir = NewDirectory(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy(); iwc.SetFlushPolicy(flushPolicy); int numDWPT = 1 + Random().Next(8); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT); iwc.SetIndexerThreadPool(threadPool); IndexWriter writer = new IndexWriter(dir, iwc); flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy; DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.FlushControl; Assert.AreEqual(0, flushControl.FlushBytes(), " bytes must be 0 after init"); IndexThread[] threads = new IndexThread[numThreads]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, true); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due"); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs()); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); if (flushPolicy.FlushOnRAM() && !flushPolicy.FlushOnDocCount() && !flushPolicy.FlushOnDeleteTerms()) { long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0); Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark"); if (flushPolicy.HasMarkedPending) { assertTrue("max: " + maxRAMBytes + " " + flushControl.PeakActiveBytes, maxRAMBytes <= flushControl.PeakActiveBytes); } } AssertActiveBytesAfter(flushControl); writer.Commit(); Assert.AreEqual(0, flushControl.ActiveBytes()); IndexReader r = DirectoryReader.Open(dir); Assert.AreEqual(numDocumentsToIndex, r.NumDocs); Assert.AreEqual(numDocumentsToIndex, r.MaxDoc); if (!flushPolicy.FlushOnRAM()) { assertFalse("never stall if we don't flush on RAM", docsWriter.FlushControl.StallControl.WasStalled()); assertFalse("never block if we don't flush on RAM", docsWriter.FlushControl.StallControl.HasBlocked()); } r.Dispose(); writer.Dispose(); dir.Dispose(); }
protected internal PayloadFilter(AtomicInteger payloadCount, TokenStream input) : base(input) { this.PayloadCount = payloadCount; PayloadAtt = AddAttribute<IPayloadAttribute>(); }
public virtual void TestStallControl() { int[] numThreads = new int[] { 4 + Random().Next(8), 1 }; int numDocumentsToIndex = 50 + Random().Next(50); for (int i = 0; i < numThreads.Length; i++) { AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex); MockDirectoryWrapper dir = NewMockDirectory(); // mock a very slow harddisk sometimes here so that flushing is very slow dir.Throttling = MockDirectoryWrapper.Throttling_e.SOMETIMES; IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH); FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy(); iwc.SetFlushPolicy(flushPolicy); DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numThreads[i] == 1 ? 1 : 2); iwc.SetIndexerThreadPool(threadPool); // with such a small ram buffer we should be stalled quiet quickly iwc.SetRAMBufferSizeMB(0.25); IndexWriter writer = new IndexWriter(dir, iwc); IndexThread[] threads = new IndexThread[numThreads[i]]; for (int x = 0; x < threads.Length; x++) { threads[x] = new IndexThread(this, numDocs, numThreads[i], writer, LineDocFile, false); threads[x].Start(); } for (int x = 0; x < threads.Length; x++) { threads[x].Join(); } DocumentsWriter docsWriter = writer.DocsWriter; Assert.IsNotNull(docsWriter); DocumentsWriterFlushControl flushControl = docsWriter.FlushControl; Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due"); Assert.AreEqual(numDocumentsToIndex, writer.NumDocs()); Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc); if (numThreads[i] == 1) { assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.FlushControl.StallControl.HasBlocked()); } if (docsWriter.FlushControl.PeakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d)) { Assert.IsTrue(docsWriter.FlushControl.StallControl.WasStalled()); } AssertActiveBytesAfter(flushControl); writer.Dispose(true); dir.Dispose(); } }
public virtual void TestCloseUnderException() { Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); writer.Commit(); writer.Dispose(); DirectoryReader open = DirectoryReader.Open(dir); bool throwOnClose = !Rarely(); AtomicReader wrap = SlowCompositeReaderWrapper.Wrap(open); FilterAtomicReader reader = new FilterAtomicReaderAnonymousInnerClassHelper(this, wrap, throwOnClose); IList<IndexReader.ReaderClosedListener> listeners = new List<IndexReader.ReaderClosedListener>(); int listenerCount = Random().Next(20); AtomicInteger count = new AtomicInteger(); bool faultySet = false; for (int i = 0; i < listenerCount; i++) { if (Rarely()) { faultySet = true; reader.AddReaderClosedListener(new FaultyListener()); } else { count.IncrementAndGet(); reader.AddReaderClosedListener(new CountListener(count)); } } if (!faultySet && !throwOnClose) { reader.AddReaderClosedListener(new FaultyListener()); } try { reader.Dispose(); Assert.Fail("expected Exception"); } catch (InvalidOperationException ex) { if (throwOnClose) { Assert.AreEqual("BOOM!", ex.Message); } else { Assert.AreEqual("GRRRRRRRRRRRR!", ex.Message); } } try { var aaa = reader.Fields; Assert.Fail("we are closed"); } catch (AlreadyClosedException ex) { } if (Random().NextBoolean()) { reader.Dispose(); // call it again } Assert.AreEqual(0, count.Get()); wrap.Dispose(); dir.Dispose(); }
public IndexThread(TestFlushByRamOrCountsPolicy outerInstance, AtomicInteger pendingDocs, int numThreads, IndexWriter writer, LineFileDocs docs, bool doRandomCommit) { this.OuterInstance = outerInstance; this.PendingDocs = pendingDocs; this.Writer = writer; Iwc = writer.Config; this.Docs = docs; this.DoRandomCommit = doRandomCommit; }
public virtual void TestStressMultiThreading() { Directory dir = NewDirectory(); IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); IndexWriter writer = new IndexWriter(dir, conf); // create index int numThreads = TestUtil.NextInt(Random(), 3, 6); int numDocs = AtLeast(2000); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); doc.Add(new StringField("id", "doc" + i, Store.NO)); double group = Random().NextDouble(); string g; if (group < 0.1) { g = "g0"; } else if (group < 0.5) { g = "g1"; } else if (group < 0.8) { g = "g2"; } else { g = "g3"; } doc.Add(new StringField("updKey", g, Store.NO)); for (int j = 0; j < numThreads; j++) { long value = Random().Next(); doc.Add(new NumericDocValuesField("f" + j, value)); doc.Add(new NumericDocValuesField("cf" + j, value * 2)); // control, always updated to f * 2 } writer.AddDocument(doc); } CountDownLatch done = new CountDownLatch(numThreads); AtomicInteger numUpdates = new AtomicInteger(AtLeast(100)); // same thread updates a field as well as reopens ThreadClass[] threads = new ThreadClass[numThreads]; for (int i = 0; i < threads.Length; i++) { string f = "f" + i; string cf = "cf" + i; threads[i] = new ThreadAnonymousInnerClassHelper(this, "UpdateThread-" + i, writer, numDocs, done, numUpdates, f, cf); } foreach (ThreadClass t in threads) { t.Start(); } done.@await(); writer.Dispose(); DirectoryReader reader = DirectoryReader.Open(dir); foreach (AtomicReaderContext context in reader.Leaves) { AtomicReader r = context.AtomicReader; for (int i = 0; i < numThreads; i++) { NumericDocValues ndv = r.GetNumericDocValues("f" + i); NumericDocValues control = r.GetNumericDocValues("cf" + i); Bits docsWithNdv = r.GetDocsWithField("f" + i); Bits docsWithControl = r.GetDocsWithField("cf" + i); Bits liveDocs = r.LiveDocs; for (int j = 0; j < r.MaxDoc; j++) { if (liveDocs == null || liveDocs.Get(j)) { Assert.AreEqual(docsWithNdv.Get(j), docsWithControl.Get(j)); if (docsWithNdv.Get(j)) { Assert.AreEqual(control.Get(j), ndv.Get(j) * 2); } } } } } reader.Dispose(); dir.Dispose(); }
public virtual void TestGetDocsWithFieldThreadSafety() { IFieldCache cache = FieldCache.DEFAULT; cache.PurgeAllCaches(); int NUM_THREADS = 3; ThreadClass[] threads = new ThreadClass[NUM_THREADS]; AtomicBoolean failed = new AtomicBoolean(); AtomicInteger iters = new AtomicInteger(); int NUM_ITER = 200 * RANDOM_MULTIPLIER; Barrier restart = new Barrier(NUM_THREADS, (barrier) => new RunnableAnonymousInnerClassHelper(this, cache, iters).Run()); for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++) { threads[threadIDX] = new ThreadAnonymousInnerClassHelper(this, cache, failed, iters, NUM_ITER, restart); threads[threadIDX].Start(); } for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++) { threads[threadIDX].Join(); } Assert.IsFalse(failed.Get()); }
public ThreadAnonymousInnerClassHelper(TestDirectoryTaxonomyWriter outerInstance, int range, AtomicInteger numCats, ConcurrentDictionary<string, string> values, Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter tw) { this.outerInstance = outerInstance; this.range = range; this.numCats = numCats; this.values = values; this.tw = tw; }
public CollectorAnonymousInnerClassHelper(TestBooleanOr outerInstance, BulkScorer scorer, FixedBitSet hits, AtomicInteger end) { this.OuterInstance = outerInstance; this.scorer = scorer; this.Hits = hits; this.End = end; }
public ConcurrentMergeSchedulerAnonymousInnerClassHelper(TestConcurrentMergeScheduler outerInstance, int maxMergeCount, CountdownEvent enoughMergesWaiting, AtomicInteger runningMergeCount, AtomicBoolean failed) { this.OuterInstance = outerInstance; this.MaxMergeCount = maxMergeCount; this.EnoughMergesWaiting = enoughMergesWaiting; this.RunningMergeCount = runningMergeCount; this.Failed = failed; }
public RunnableAnonymousInnerClassHelper(TestFieldCache outerInstance, IFieldCache cache, AtomicInteger iters) { this.OuterInstance = outerInstance; this.Cache = cache; this.Iters = iters; }