public virtual void TestUpdateDelteSlices() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random().Next(); } DeleteSlice slice1 = queue.NewSlice(); DeleteSlice slice2 = queue.NewSlice(); BufferedUpdates bd1 = new BufferedUpdates(); BufferedUpdates bd2 = new BufferedUpdates(); int last1 = 0; int last2 = 0; HashSet<Term> uniqueValues = new HashSet<Term>(); for (int j = 0; j < ids.Length; j++) { int? i = ids[j]; // create an array here since we compare identity below against tailItem Term[] term = new Term[] { new Term("id", i.ToString()) }; uniqueValues.Add(term[0]); queue.AddDelete(term); if (Random().Next(20) == 0 || j == ids.Length - 1) { queue.UpdateSlice(slice1); Assert.IsTrue(slice1.IsTailItem(term)); slice1.Apply(bd1, j); AssertAllBetween(last1, j, bd1, ids); last1 = j + 1; } if (Random().Next(10) == 5 || j == ids.Length - 1) { queue.UpdateSlice(slice2); Assert.IsTrue(slice2.IsTailItem(term)); slice2.Apply(bd2, j); AssertAllBetween(last2, j, bd2, ids); last2 = j + 1; } Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes()); } Assert.AreEqual(uniqueValues, bd1.Terms_Nunit().Keys); Assert.AreEqual(uniqueValues, bd2.Terms_Nunit().Keys); HashSet<Term> frozenSet = new HashSet<Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes()); frozenSet.Add(new Term(t.Field(), bytesRef)); } Assert.AreEqual(uniqueValues, frozenSet); Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze"); }
public virtual void TestStressDeleteQueue() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); HashSet <Term> uniqueValues = new HashSet <Term>(); int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random().Next(); uniqueValues.Add(new Term("id", ids[i].ToString())); } CountDownLatch latch = new CountDownLatch(1); AtomicInteger index = new AtomicInteger(0); int numThreads = 2 + Random().Next(5); UpdateThread[] threads = new UpdateThread[numThreads]; for (int i = 0; i < threads.Length; i++) { threads[i] = new UpdateThread(queue, index, ids, latch); threads[i].Start(); } latch.countDown(); for (int i = 0; i < threads.Length; i++) { threads[i].Join(); } foreach (UpdateThread updateThread in threads) { DeleteSlice slice = updateThread.Slice; queue.UpdateSlice(slice); BufferedUpdates deletes = updateThread.Deletes; slice.Apply(deletes, BufferedUpdates.MAX_INT); Assert.AreEqual(uniqueValues, deletes.Terms_Nunit().Keys); } queue.TryApplyGlobalSlice(); HashSet <Term> frozenSet = new HashSet <Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes()); frozenSet.Add(new Term(t.Field(), bytesRef)); } Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze"); Assert.AreEqual(uniqueValues.Count, frozenSet.Count); Assert.AreEqual(uniqueValues, frozenSet); }
public virtual void TestStressDeleteQueue() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); ISet <Term> uniqueValues = new JCG.HashSet <Term>(); int size = 10000 + Random.Next(500) * RandomMultiplier; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random.Next(); uniqueValues.Add(new Term("id", ids[i].ToString())); } CountdownEvent latch = new CountdownEvent(1); AtomicInt32 index = new AtomicInt32(0); int numThreads = 2 + Random.Next(5); UpdateThread[] threads = new UpdateThread[numThreads]; for (int i = 0; i < threads.Length; i++) { threads[i] = new UpdateThread(queue, index, ids, latch); threads[i].Start(); } latch.Signal(); for (int i = 0; i < threads.Length; i++) { threads[i].Join(); } foreach (UpdateThread updateThread in threads) { DeleteSlice slice = updateThread.slice; queue.UpdateSlice(slice); BufferedUpdates deletes = updateThread.deletes; slice.Apply(deletes, BufferedUpdates.MAX_INT32); assertEquals(uniqueValues, deletes.terms.Keys); } queue.TryApplyGlobalSlice(); ISet <Term> frozenSet = new JCG.HashSet <Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes); frozenSet.Add(new Term(t.Field, bytesRef)); } Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze"); Assert.AreEqual(uniqueValues.Count, frozenSet.Count); assertEquals(uniqueValues, frozenSet); }
internal FlushedSegment(SegmentCommitInfo segmentInfo, FieldInfos fieldInfos, BufferedUpdates segmentUpdates, MutableBits liveDocs, int delCount) { this.SegmentInfo = segmentInfo; this.FieldInfos = fieldInfos; this.SegmentUpdates = segmentUpdates != null && segmentUpdates.Any() ? new FrozenBufferedUpdates(segmentUpdates, true) : null; this.LiveDocs = liveDocs; this.DelCount = delCount; }
public DocumentsWriterPerThread(string segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue, FieldInfos.Builder fieldInfos) { this.DirectoryOrig = directory; this.Directory = new TrackingDirectoryWrapper(directory); this.FieldInfos = fieldInfos; this.IndexWriterConfig = indexWriterConfig; this.InfoStream = infoStream; this.Codec = indexWriterConfig.Codec; this.docState = new DocState(this, infoStream); this.docState.Similarity = indexWriterConfig.Similarity; bytesUsed = Counter.NewCounter(); ByteBlockAllocator = new DirectTrackingAllocator(bytesUsed); PendingUpdates = new BufferedUpdates(); intBlockAllocator = new IntBlockAllocator(bytesUsed); this.DeleteQueue = deleteQueue; Debug.Assert(numDocsInRAM == 0, "num docs " + numDocsInRAM); PendingUpdates.Clear(); DeleteSlice = deleteQueue.NewSlice(); SegmentInfo_Renamed = new SegmentInfo(DirectoryOrig, Constants.LUCENE_MAIN_VERSION, segmentName, -1, false, Codec, null); Debug.Assert(numDocsInRAM == 0); if (INFO_VERBOSE && infoStream.IsEnabled("DWPT")) { infoStream.Message("DWPT", Thread.CurrentThread.Name + " init seg=" + segmentName + " delQueue=" + deleteQueue); } // this should be the last call in the ctor // it really sucks that we need to pull this within the ctor and pass this ref to the chain! Consumer = indexWriterConfig.IndexingChain.GetChain(this); }
/// <summary> /// Create a shallow copy of <seealso cref="SegmentWriteState"/> with a new segment suffix. </summary> public SegmentWriteState(SegmentWriteState state, string segmentSuffix) { InfoStream = state.InfoStream; Directory = state.Directory; SegmentInfo = state.SegmentInfo; FieldInfos = state.FieldInfos; TermIndexInterval = state.TermIndexInterval; Context = state.Context; this.SegmentSuffix = segmentSuffix; SegUpdates = state.SegUpdates; DelCountOnFlush = state.DelCountOnFlush; }
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto) { bufferedUpdates.AddNumericUpdate((NumericDocValuesUpdate)item, docIDUpto); }
//internal static readonly AtomicReferenceFieldUpdater<Node, Node> NextUpdater = AtomicReferenceFieldUpdater.newUpdater(typeof(Node), typeof(Node), "next"); internal virtual void Apply(BufferedUpdates bufferedDeletes, int docIDUpto) { throw new InvalidOperationException("sentinel item must never be applied"); }
internal FlushedSegment(SegmentCommitInfo segmentInfo, FieldInfos fieldInfos, BufferedUpdates segmentUpdates, IMutableBits liveDocs, int delCount) { this.segmentInfo = segmentInfo; this.fieldInfos = fieldInfos; this.segmentUpdates = segmentUpdates != null && segmentUpdates.Any() ? new FrozenBufferedUpdates(segmentUpdates, true) : null; this.liveDocs = liveDocs; this.delCount = delCount; }
/// <summary> /// Sole constructor. </summary> public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context) : this(infoStream, directory, segmentInfo, fieldInfos, termIndexInterval, segUpdates, context, "") { }
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto) { bufferedUpdates.AddNumericUpdate((NumericDocValuesUpdate)Item, docIDUpto); }
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto) { foreach (Term term in (Term[])Item) { bufferedUpdates.AddTerm(term, docIDUpto); } }
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto) { foreach (Query query in (Query[])Item) { bufferedUpdates.AddQuery(query, docIDUpto); } }
internal override void Apply(BufferedUpdates bufferedDeletes, int docIDUpto) { bufferedDeletes.AddTerm((Term)Item, docIDUpto); }
public virtual void Apply(BufferedUpdates del, int docIDUpto) { if (SliceHead == SliceTail) { // 0 length slice return; } /* * When we apply a slice we take the head and get its next as our first * item to apply and continue until we applied the tail. If the head and * tail in this slice are not equal then there will be at least one more * non-null node in the slice! */ Node current = SliceHead; do { current = current.Next; Debug.Assert(current != null, "slice property violated between the head on the tail must not be a null node"); current.Apply(del, docIDUpto); // System.out.println(Thread.currentThread().getName() + ": pull " + current + " docIDUpto=" + docIDUpto); } while (current != SliceTail); Reset(); }
internal readonly bool IsSegmentPrivate; // set to true iff this frozen packet represents // a segment private deletes. in that case is should // only have Queries public FrozenBufferedUpdates(BufferedUpdates deletes, bool isSegmentPrivate) { this.IsSegmentPrivate = isSegmentPrivate; Debug.Assert(!isSegmentPrivate || deletes.Terms.Count == 0, "segment private package should only have del queries"); Term[] termsArray = deletes.Terms.Keys.ToArray(/*new Term[deletes.Terms.Count]*/); termCount = termsArray.Length; ArrayUtil.TimSort(termsArray); PrefixCodedTerms.Builder builder = new PrefixCodedTerms.Builder(); foreach (Term term in termsArray) { builder.Add(term); } Terms = builder.Finish(); Queries = new Query[deletes.Queries.Count]; QueryLimits = new int?[deletes.Queries.Count]; int upto = 0; foreach (KeyValuePair <Query, int?> ent in deletes.Queries) { Queries[upto] = ent.Key; QueryLimits[upto] = ent.Value; upto++; } // TODO if a Term affects multiple fields, we could keep the updates key'd by Term // so that it maps to all fields it affects, sorted by their docUpto, and traverse // that Term only once, applying the update to all fields that still need to be // updated. IList <NumericDocValuesUpdate> allNumericUpdates = new List <NumericDocValuesUpdate>(); int numericUpdatesSize = 0; foreach (/*Linked*/ HashMap <Term, NumericDocValuesUpdate> numericUpdates in deletes.NumericUpdates.Values) { foreach (NumericDocValuesUpdate update in numericUpdates.Values) { allNumericUpdates.Add(update); numericUpdatesSize += update.SizeInBytes(); } } NumericDVUpdates = allNumericUpdates.ToArray(); // TODO if a Term affects multiple fields, we could keep the updates key'd by Term // so that it maps to all fields it affects, sorted by their docUpto, and traverse // that Term only once, applying the update to all fields that still need to be // updated. IList <BinaryDocValuesUpdate> allBinaryUpdates = new List <BinaryDocValuesUpdate>(); int binaryUpdatesSize = 0; foreach (/*Linked*/ HashMap <Term, BinaryDocValuesUpdate> binaryUpdates in deletes.BinaryUpdates.Values) { foreach (BinaryDocValuesUpdate update in binaryUpdates.Values) { allBinaryUpdates.Add(update); binaryUpdatesSize += update.SizeInBytes(); } } BinaryDVUpdates = allBinaryUpdates.ToArray(); BytesUsed = (int)Terms.SizeInBytes + Queries.Length * BYTES_PER_DEL_QUERY + numericUpdatesSize + NumericDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF + binaryUpdatesSize + BinaryDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF; NumTermDeletes = deletes.NumTermDeletes.Get(); }
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto) { bufferedUpdates.AddBinaryUpdate((BinaryDocValuesUpdate)Item, docIDUpto); }
//internal static readonly AtomicReferenceFieldUpdater<Node, Node> NextUpdater = AtomicReferenceFieldUpdater.newUpdater(typeof(Node), typeof(Node), "next"); internal virtual void Apply(BufferedUpdates bufferedDeletes, int docIDUpto) { throw IllegalStateException.Create("sentinel item must never be applied"); }
public DocumentsWriterDeleteQueue(BufferedUpdates globalBufferedUpdates, long generation) { this.GlobalBufferedUpdates = globalBufferedUpdates; this.Generation = generation; /* * we use a sentinel instance as our initial tail. No slice will ever try to * apply this tail since the head is always omitted. */ Tail = new Node(null); // sentinel GlobalSlice = new DeleteSlice(Tail); }
/// <summary> /// Constructor which takes segment suffix. /// </summary> /// <seealso cref="SegmentWriteState(InfoStream, Directory, SegmentInfo, FieldInfos, int, /// BufferedUpdates, IOContext)"/> public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context, string segmentSuffix) { this.InfoStream = infoStream; this.SegUpdates = segUpdates; this.Directory = directory; this.SegmentInfo = segmentInfo; this.FieldInfos = fieldInfos; this.TermIndexInterval = termIndexInterval; this.SegmentSuffix = segmentSuffix; this.Context = context; }
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInteger index, int?[] ids, CountDownLatch latch) { this.Queue = queue; this.Index = index; this.Ids = ids; this.Slice = queue.NewSlice(); Deletes = new BufferedUpdates(); this.Latch = latch; }
internal readonly bool isSegmentPrivate; // set to true iff this frozen packet represents // a segment private deletes. in that case is should // only have Queries public FrozenBufferedUpdates(BufferedUpdates deletes, bool isSegmentPrivate) { this.isSegmentPrivate = isSegmentPrivate; Debug.Assert(!isSegmentPrivate || deletes.terms.Count == 0, "segment private package should only have del queries"); Term[] termsArray = deletes.terms.Keys.ToArray(/*new Term[deletes.Terms.Count]*/); termCount = termsArray.Length; ArrayUtil.TimSort(termsArray); PrefixCodedTerms.Builder builder = new PrefixCodedTerms.Builder(); foreach (Term term in termsArray) { builder.Add(term); } terms = builder.Finish(); queries = new Query[deletes.queries.Count]; queryLimits = new int[deletes.queries.Count]; int upto = 0; foreach (KeyValuePair <Query, int?> ent in deletes.queries) { queries[upto] = ent.Key; if (ent.Value.HasValue) { queryLimits[upto] = ent.Value.Value; } else { // LUCENENET NOTE: According to this: http://stackoverflow.com/a/13914344 // we are supposed to throw an exception in this case, rather than // silently fail. throw new NullReferenceException(); } upto++; } // TODO if a Term affects multiple fields, we could keep the updates key'd by Term // so that it maps to all fields it affects, sorted by their docUpto, and traverse // that Term only once, applying the update to all fields that still need to be // updated. IList <NumericDocValuesUpdate> allNumericUpdates = new List <NumericDocValuesUpdate>(); int numericUpdatesSize = 0; foreach (var numericUpdates in deletes.numericUpdates.Values) { foreach (NumericDocValuesUpdate update in numericUpdates.Values) { allNumericUpdates.Add(update); numericUpdatesSize += update.GetSizeInBytes(); } } numericDVUpdates = allNumericUpdates.ToArray(); // TODO if a Term affects multiple fields, we could keep the updates key'd by Term // so that it maps to all fields it affects, sorted by their docUpto, and traverse // that Term only once, applying the update to all fields that still need to be // updated. IList <BinaryDocValuesUpdate> allBinaryUpdates = new List <BinaryDocValuesUpdate>(); int binaryUpdatesSize = 0; foreach (var binaryUpdates in deletes.binaryUpdates.Values) { foreach (BinaryDocValuesUpdate update in binaryUpdates.Values) { allBinaryUpdates.Add(update); binaryUpdatesSize += update.GetSizeInBytes(); } } binaryDVUpdates = allBinaryUpdates.ToArray(); bytesUsed = (int)terms.GetSizeInBytes() + queries.Length * BYTES_PER_DEL_QUERY + numericUpdatesSize + numericDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF + binaryUpdatesSize + binaryDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF; numTermDeletes = deletes.numTermDeletes.Get(); }
internal override void Apply(BufferedUpdates bufferedDeletes, int docIDUpto) { bufferedDeletes.AddTerm((Term)item, docIDUpto); }
private void AssertAllBetween(int start, int end, BufferedUpdates deletes, int?[] ids) { for (int i = start; i <= end; i++) { Assert.AreEqual(Convert.ToInt32(end), deletes.Terms_Nunit()[new Term("id", ids[i].ToString())]); } }
internal override void Apply(BufferedUpdates bufferedUpdates, int docIDUpto) { bufferedUpdates.AddBinaryUpdate((BinaryDocValuesUpdate)item, docIDUpto); }
/// <summary> /// Constructor which takes segment suffix. /// </summary> /// <seealso cref= #SegmentWriteState(InfoStream, Directory, SegmentInfo, FieldInfos, int, /// BufferedUpdates, IOContext) </seealso> public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, int termIndexInterval, BufferedUpdates segUpdates, IOContext context, string segmentSuffix) { this.InfoStream = infoStream; this.SegUpdates = segUpdates; this.Directory = directory; this.SegmentInfo = segmentInfo; this.FieldInfos = fieldInfos; this.TermIndexInterval = termIndexInterval; this.SegmentSuffix = segmentSuffix; this.Context = context; }