public virtual void TestUpdateDelteSlices() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); int size = 200 + Random.Next(500) * RANDOM_MULTIPLIER; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random.Next(); } DeleteSlice slice1 = queue.NewSlice(); DeleteSlice slice2 = queue.NewSlice(); BufferedUpdates bd1 = new BufferedUpdates(); BufferedUpdates bd2 = new BufferedUpdates(); int last1 = 0; int last2 = 0; ISet <Term> uniqueValues = new JCG.HashSet <Term>(); for (int j = 0; j < ids.Length; j++) { int?i = ids[j]; // create an array here since we compare identity below against tailItem Term[] term = new Term[] { new Term("id", i.ToString()) }; uniqueValues.Add(term[0]); queue.AddDelete(term); if (Random.Next(20) == 0 || j == ids.Length - 1) { queue.UpdateSlice(slice1); Assert.IsTrue(slice1.IsTailItem(term)); slice1.Apply(bd1, j); AssertAllBetween(last1, j, bd1, ids); last1 = j + 1; } if (Random.Next(10) == 5 || j == ids.Length - 1) { queue.UpdateSlice(slice2); Assert.IsTrue(slice2.IsTailItem(term)); slice2.Apply(bd2, j); AssertAllBetween(last2, j, bd2, ids); last2 = j + 1; } Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes); } assertEquals(uniqueValues, new JCG.HashSet <Term>(bd1.terms.Keys)); assertEquals(uniqueValues, new JCG.HashSet <Term>(bd2.terms.Keys)); var frozenSet = new JCG.HashSet <Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes); frozenSet.Add(new Term(t.Field, bytesRef)); } assertEquals(uniqueValues, frozenSet); Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze"); }
public virtual void TestUpdateDelteSlices() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random().Next(); } DeleteSlice slice1 = queue.NewSlice(); DeleteSlice slice2 = queue.NewSlice(); BufferedUpdates bd1 = new BufferedUpdates(); BufferedUpdates bd2 = new BufferedUpdates(); int last1 = 0; int last2 = 0; HashSet<Term> uniqueValues = new HashSet<Term>(); for (int j = 0; j < ids.Length; j++) { int? i = ids[j]; // create an array here since we compare identity below against tailItem Term[] term = new Term[] { new Term("id", i.ToString()) }; uniqueValues.Add(term[0]); queue.AddDelete(term); if (Random().Next(20) == 0 || j == ids.Length - 1) { queue.UpdateSlice(slice1); Assert.IsTrue(slice1.IsTailItem(term)); slice1.Apply(bd1, j); AssertAllBetween(last1, j, bd1, ids); last1 = j + 1; } if (Random().Next(10) == 5 || j == ids.Length - 1) { queue.UpdateSlice(slice2); Assert.IsTrue(slice2.IsTailItem(term)); slice2.Apply(bd2, j); AssertAllBetween(last2, j, bd2, ids); last2 = j + 1; } Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes()); } Assert.AreEqual(uniqueValues, bd1.Terms_Nunit().Keys); Assert.AreEqual(uniqueValues, bd2.Terms_Nunit().Keys); HashSet<Term> frozenSet = new HashSet<Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes()); frozenSet.Add(new Term(t.Field(), bytesRef)); } Assert.AreEqual(uniqueValues, frozenSet); Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze"); }
public DocumentsWriterPerThread(string segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue, FieldInfos.Builder fieldInfos) { this.directoryOrig = directory; this.directory = new TrackingDirectoryWrapper(directory); this.fieldInfos = fieldInfos; this.indexWriterConfig = indexWriterConfig; this.infoStream = infoStream; this.codec = indexWriterConfig.Codec; this.docState = new DocState(this, infoStream); this.docState.similarity = indexWriterConfig.Similarity; bytesUsed = Counter.NewCounter(); byteBlockAllocator = new DirectTrackingAllocator(bytesUsed); pendingUpdates = new BufferedUpdates(); intBlockAllocator = new Int32BlockAllocator(bytesUsed); this.deleteQueue = deleteQueue; if (Debugging.AssertsEnabled) { Debugging.Assert(numDocsInRAM == 0, "num docs {0}", numDocsInRAM); } pendingUpdates.Clear(); deleteSlice = deleteQueue.NewSlice(); segmentInfo = new SegmentInfo(directoryOrig, Constants.LUCENE_MAIN_VERSION, segmentName, -1, false, codec, null); if (Debugging.AssertsEnabled) { Debugging.Assert(numDocsInRAM == 0); } if (INFO_VERBOSE && infoStream.IsEnabled("DWPT")) { infoStream.Message("DWPT", Thread.CurrentThread.Name + " init seg=" + segmentName + " delQueue=" + deleteQueue); } // this should be the last call in the ctor // it really sucks that we need to pull this within the ctor and pass this ref to the chain! consumer = indexWriterConfig.IndexingChain.GetChain(this); }
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInt32 index, int?[] ids, CountdownEvent latch) { this.Queue = queue; this.Index = index; this.Ids = ids; this.Slice = queue.NewSlice(); Deletes = new BufferedUpdates(); this.Latch = latch; }
public DocumentsWriterPerThread(string segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue, FieldInfos.Builder fieldInfos) { this.DirectoryOrig = directory; this.Directory = new TrackingDirectoryWrapper(directory); this.FieldInfos = fieldInfos; this.IndexWriterConfig = indexWriterConfig; this.InfoStream = infoStream; this.Codec = indexWriterConfig.Codec; this.docState = new DocState(this, infoStream); this.docState.Similarity = indexWriterConfig.Similarity; bytesUsed = Counter.NewCounter(); ByteBlockAllocator = new DirectTrackingAllocator(bytesUsed); PendingUpdates = new BufferedUpdates(); intBlockAllocator = new IntBlockAllocator(bytesUsed); this.DeleteQueue = deleteQueue; Debug.Assert(numDocsInRAM == 0, "num docs " + numDocsInRAM); PendingUpdates.Clear(); DeleteSlice = deleteQueue.NewSlice(); SegmentInfo_Renamed = new SegmentInfo(DirectoryOrig, Constants.LUCENE_MAIN_VERSION, segmentName, -1, false, Codec, null); Debug.Assert(numDocsInRAM == 0); if (INFO_VERBOSE && infoStream.IsEnabled("DWPT")) { infoStream.Message("DWPT", Thread.CurrentThread.Name + " init seg=" + segmentName + " delQueue=" + deleteQueue); } // this should be the last call in the ctor // it really sucks that we need to pull this within the ctor and pass this ref to the chain! Consumer = indexWriterConfig.IndexingChain.GetChain(this); }
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInteger index, int?[] ids, CountDownLatch latch) { this.Queue = queue; this.Index = index; this.Ids = ids; this.Slice = queue.NewSlice(); Deletes = new BufferedUpdates(); this.Latch = latch; }