public virtual void TestUpdateDelteSlices() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); int size = 200 + Random.Next(500) * RANDOM_MULTIPLIER; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random.Next(); } DeleteSlice slice1 = queue.NewSlice(); DeleteSlice slice2 = queue.NewSlice(); BufferedUpdates bd1 = new BufferedUpdates(); BufferedUpdates bd2 = new BufferedUpdates(); int last1 = 0; int last2 = 0; ISet <Term> uniqueValues = new JCG.HashSet <Term>(); for (int j = 0; j < ids.Length; j++) { int?i = ids[j]; // create an array here since we compare identity below against tailItem Term[] term = new Term[] { new Term("id", i.ToString()) }; uniqueValues.Add(term[0]); queue.AddDelete(term); if (Random.Next(20) == 0 || j == ids.Length - 1) { queue.UpdateSlice(slice1); Assert.IsTrue(slice1.IsTailItem(term)); slice1.Apply(bd1, j); AssertAllBetween(last1, j, bd1, ids); last1 = j + 1; } if (Random.Next(10) == 5 || j == ids.Length - 1) { queue.UpdateSlice(slice2); Assert.IsTrue(slice2.IsTailItem(term)); slice2.Apply(bd2, j); AssertAllBetween(last2, j, bd2, ids); last2 = j + 1; } Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes); } assertEquals(uniqueValues, new JCG.HashSet <Term>(bd1.terms.Keys)); assertEquals(uniqueValues, new JCG.HashSet <Term>(bd2.terms.Keys)); var frozenSet = new JCG.HashSet <Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes); frozenSet.Add(new Term(t.Field, bytesRef)); } assertEquals(uniqueValues, frozenSet); Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze"); }
public virtual void TestUpdateDelteSlices() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random().Next(); } DeleteSlice slice1 = queue.NewSlice(); DeleteSlice slice2 = queue.NewSlice(); BufferedUpdates bd1 = new BufferedUpdates(); BufferedUpdates bd2 = new BufferedUpdates(); int last1 = 0; int last2 = 0; HashSet<Term> uniqueValues = new HashSet<Term>(); for (int j = 0; j < ids.Length; j++) { int? i = ids[j]; // create an array here since we compare identity below against tailItem Term[] term = new Term[] { new Term("id", i.ToString()) }; uniqueValues.Add(term[0]); queue.AddDelete(term); if (Random().Next(20) == 0 || j == ids.Length - 1) { queue.UpdateSlice(slice1); Assert.IsTrue(slice1.IsTailItem(term)); slice1.Apply(bd1, j); AssertAllBetween(last1, j, bd1, ids); last1 = j + 1; } if (Random().Next(10) == 5 || j == ids.Length - 1) { queue.UpdateSlice(slice2); Assert.IsTrue(slice2.IsTailItem(term)); slice2.Apply(bd2, j); AssertAllBetween(last2, j, bd2, ids); last2 = j + 1; } Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes()); } Assert.AreEqual(uniqueValues, bd1.Terms_Nunit().Keys); Assert.AreEqual(uniqueValues, bd2.Terms_Nunit().Keys); HashSet<Term> frozenSet = new HashSet<Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes()); frozenSet.Add(new Term(t.Field(), bytesRef)); } Assert.AreEqual(uniqueValues, frozenSet); Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze"); }
private void FinishDocument(Term delTerm) { /* * here we actually finish the document in two steps 1. push the delete into * the queue and update our slice. 2. increment the DWPT private document * id. * * the updated slice we get from 1. holds all the deletes that have occurred * since we updated the slice the last time. */ bool applySlice = numDocsInRAM != 0; if (delTerm != null) { deleteQueue.Add(delTerm, deleteSlice); if (Debugging.AssertsEnabled) { Debugging.Assert(deleteSlice.IsTailItem(delTerm), "expected the delete term as the tail item"); } } else { applySlice &= deleteQueue.UpdateSlice(deleteSlice); } if (applySlice) { deleteSlice.Apply(pendingUpdates, numDocsInRAM); } // if we don't need to apply we must reset! else { deleteSlice.Reset(); } ++numDocsInRAM; }
public virtual void TestStressDeleteQueue() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); ISet <Term> uniqueValues = new JCG.HashSet <Term>(); int size = 10000 + Random.Next(500) * RANDOM_MULTIPLIER; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random.Next(); uniqueValues.Add(new Term("id", ids[i].ToString())); } CountdownEvent latch = new CountdownEvent(1); AtomicInt32 index = new AtomicInt32(0); int numThreads = 2 + Random.Next(5); UpdateThread[] threads = new UpdateThread[numThreads]; for (int i = 0; i < threads.Length; i++) { threads[i] = new UpdateThread(queue, index, ids, latch); threads[i].Start(); } latch.Signal(); for (int i = 0; i < threads.Length; i++) { threads[i].Join(); } foreach (UpdateThread updateThread in threads) { DeleteSlice slice = updateThread.Slice; queue.UpdateSlice(slice); BufferedUpdates deletes = updateThread.Deletes; slice.Apply(deletes, BufferedUpdates.MAX_INT32); assertEquals(uniqueValues, new JCG.HashSet <Term>(deletes.terms.Keys)); } queue.TryApplyGlobalSlice(); ISet <Term> frozenSet = new JCG.HashSet <Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes); frozenSet.Add(new Term(t.Field, bytesRef)); } Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze"); Assert.AreEqual(uniqueValues.Count, frozenSet.Count); assertEquals(uniqueValues, frozenSet); }
public virtual void TestStressDeleteQueue() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); HashSet<Term> uniqueValues = new HashSet<Term>(); int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random().Next(); uniqueValues.Add(new Term("id", ids[i].ToString())); } CountDownLatch latch = new CountDownLatch(1); AtomicInteger index = new AtomicInteger(0); int numThreads = 2 + Random().Next(5); UpdateThread[] threads = new UpdateThread[numThreads]; for (int i = 0; i < threads.Length; i++) { threads[i] = new UpdateThread(queue, index, ids, latch); threads[i].Start(); } latch.countDown(); for (int i = 0; i < threads.Length; i++) { threads[i].Join(); } foreach (UpdateThread updateThread in threads) { DeleteSlice slice = updateThread.Slice; queue.UpdateSlice(slice); BufferedUpdates deletes = updateThread.Deletes; slice.Apply(deletes, BufferedUpdates.MAX_INT); Assert.AreEqual(uniqueValues, deletes.Terms_Nunit().Keys); } queue.TryApplyGlobalSlice(); HashSet<Term> frozenSet = new HashSet<Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes()); frozenSet.Add(new Term(t.Field(), bytesRef)); } Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze"); Assert.AreEqual(uniqueValues.Count, frozenSet.Count); Assert.AreEqual(uniqueValues, frozenSet); }