DocumentsWriterDeleteQueue is a non-blocking linked pending deletes queue. In contrast to other queue implementation we only maintain the tail of the queue. A delete queue is always used in a context of a set of DWPTs and a global delete pool. Each of the DWPT and the global pool need to maintain their 'own' head of the queue (as a DeleteSlice instance per DWPT). The difference between the DWPT and the global pool is that the DWPT starts maintaining a head once it has added its first document since for its segments private deletes only the deletes after that document are relevant. The global pool instead starts maintaining the head once this instance is created by taking the sentinel instance as its initial head.

Since each DeleteSlice maintains its own head and the list is only single linked the garbage collector takes care of pruning the list for us. All nodes in the list that are still relevant should be either directly or indirectly referenced by one of the DWPT's private DeleteSlice or by the global BufferedUpdates slice.

Each DWPT as well as the global delete pool maintain their private DeleteSlice instance. In the DWPT case updating a slice is equivalent to atomically finishing the document. The slice update guarantees a "happens before" relationship to all other updates in the same indexing session. When a DWPT updates a document it:

  1. consumes a document and finishes its processing
  2. updates its private DeleteSlice either by calling #updateSlice(DeleteSlice) or #add(Term, DeleteSlice) (if the document has a delTerm)
  3. applies all deletes in the slice to its private BufferedUpdates and resets it
  4. increments its internal document id
The DWPT also doesn't apply its current documents delete term until it has updated its delete slice which ensures the consistency of the update. If the update fails before the DeleteSlice could have been updated the deleteTerm will also not be added to its private deletes neither to the global deletes.
Example #1
0
        public virtual void TestClear()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();

            Assert.IsFalse(queue.AnyChanges());
            queue.Clear();
            Assert.IsFalse(queue.AnyChanges());
            int size               = 200 + Random.Next(500) * RANDOM_MULTIPLIER;
            int termsSinceFreeze   = 0;
            int queriesSinceFreeze = 0;

            for (int i = 0; i < size; i++)
            {
                Term term = new Term("id", "" + i);
                if (Random.Next(10) == 0)
                {
                    queue.AddDelete(new TermQuery(term));
                    queriesSinceFreeze++;
                }
                else
                {
                    queue.AddDelete(term);
                    termsSinceFreeze++;
                }
                Assert.IsTrue(queue.AnyChanges());
                if (Random.Next(10) == 0)
                {
                    queue.Clear();
                    queue.TryApplyGlobalSlice();
                    Assert.IsFalse(queue.AnyChanges());
                }
            }
        }
 public virtual void TestClear()
 {
     DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
     Assert.IsFalse(queue.AnyChanges());
     queue.Clear();
     Assert.IsFalse(queue.AnyChanges());
     int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
     int termsSinceFreeze = 0;
     int queriesSinceFreeze = 0;
     for (int i = 0; i < size; i++)
     {
         Term term = new Term("id", "" + i);
         if (Random().Next(10) == 0)
         {
             queue.AddDelete(new TermQuery(term));
             queriesSinceFreeze++;
         }
         else
         {
             queue.AddDelete(term);
             termsSinceFreeze++;
         }
         Assert.IsTrue(queue.AnyChanges());
         if (Random().Next(10) == 0)
         {
             queue.Clear();
             queue.TryApplyGlobalSlice();
             Assert.IsFalse(queue.AnyChanges());
         }
     }
 }
Example #3
0
        public virtual void TestAnyChanges()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
            int size               = 200 + Random.Next(500) * RANDOM_MULTIPLIER;
            int termsSinceFreeze   = 0;
            int queriesSinceFreeze = 0;

            for (int i = 0; i < size; i++)
            {
                Term term = new Term("id", "" + i);
                if (Random.Next(10) == 0)
                {
                    queue.AddDelete(new TermQuery(term));
                    queriesSinceFreeze++;
                }
                else
                {
                    queue.AddDelete(term);
                    termsSinceFreeze++;
                }
                Assert.IsTrue(queue.AnyChanges());
                if (Random.Next(5) == 0)
                {
                    FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
                    Assert.AreEqual(termsSinceFreeze, freezeGlobalBuffer.termCount);
                    Assert.AreEqual(queriesSinceFreeze, ((Query[])freezeGlobalBuffer.queries.Clone()).Length);
                    queriesSinceFreeze = 0;
                    termsSinceFreeze   = 0;
                    Assert.IsFalse(queue.AnyChanges());
                }
            }
        }
Example #4
0
 internal virtual void AddDeletes(DocumentsWriterDeleteQueue deleteQueue)
 {
     UninterruptableMonitor.Enter(this);
     try
     {
         IncTickets(); // first inc the ticket count - freeze opens
         // a window for #anyChanges to fail
         bool success = false;
         try
         {
             queue.Enqueue(new GlobalDeletesTicket(deleteQueue.FreezeGlobalBuffer(null)));
             success = true;
         }
         finally
         {
             if (!success)
             {
                 DecTickets();
             }
         }
     }
     finally
     {
         UninterruptableMonitor.Exit(this);
     }
 }
 public virtual void TestAnyChanges()
 {
     DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
     int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
     int termsSinceFreeze = 0;
     int queriesSinceFreeze = 0;
     for (int i = 0; i < size; i++)
     {
         Term term = new Term("id", "" + i);
         if (Random().Next(10) == 0)
         {
             queue.AddDelete(new TermQuery(term));
             queriesSinceFreeze++;
         }
         else
         {
             queue.AddDelete(term);
             termsSinceFreeze++;
         }
         Assert.IsTrue(queue.AnyChanges());
         if (Random().Next(5) == 0)
         {
             FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
             Assert.AreEqual(termsSinceFreeze, freezeGlobalBuffer.TermCount);
             Assert.AreEqual(queriesSinceFreeze, ((Query[])freezeGlobalBuffer.Queries_Nunit()).Length);
             queriesSinceFreeze = 0;
             termsSinceFreeze = 0;
             Assert.IsFalse(queue.AnyChanges());
         }
     }
 }
        public DocumentsWriterPerThread(string segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue, FieldInfos.Builder fieldInfos)
        {
            this.directoryOrig       = directory;
            this.directory           = new TrackingDirectoryWrapper(directory);
            this.fieldInfos          = fieldInfos;
            this.indexWriterConfig   = indexWriterConfig;
            this.infoStream          = infoStream;
            this.codec               = indexWriterConfig.Codec;
            this.docState            = new DocState(this, infoStream);
            this.docState.similarity = indexWriterConfig.Similarity;
            bytesUsed          = Counter.NewCounter();
            byteBlockAllocator = new DirectTrackingAllocator(bytesUsed);
            pendingUpdates     = new BufferedUpdates();
            intBlockAllocator  = new Int32BlockAllocator(bytesUsed);
            this.deleteQueue   = deleteQueue;
            if (Debugging.AssertsEnabled)
            {
                Debugging.Assert(numDocsInRAM == 0, "num docs {0}", numDocsInRAM);
            }
            pendingUpdates.Clear();
            deleteSlice = deleteQueue.NewSlice();

            segmentInfo = new SegmentInfo(directoryOrig, Constants.LUCENE_MAIN_VERSION, segmentName, -1, false, codec, null);
            if (Debugging.AssertsEnabled)
            {
                Debugging.Assert(numDocsInRAM == 0);
            }
            if (INFO_VERBOSE && infoStream.IsEnabled("DWPT"))
            {
                infoStream.Message("DWPT", Thread.CurrentThread.Name + " init seg=" + segmentName + " delQueue=" + deleteQueue);
            }
            // this should be the last call in the ctor
            // it really sucks that we need to pull this within the ctor and pass this ref to the chain!
            consumer = indexWriterConfig.IndexingChain.GetChain(this);
        }
Example #7
0
 // for asserts
 private bool SetFlushingDeleteQueue(DocumentsWriterDeleteQueue session)
 {
     lock (this)
     {
         currentFullFlushDelQueue = session;
         return(true);
     }
 }
Example #8
0
 internal bool AssertBlockedFlushes(DocumentsWriterDeleteQueue flushingQueue)
 {
     foreach (BlockedFlush blockedFlush in blockedFlushes)
     {
         Debug.Assert(blockedFlush.Dwpt.deleteQueue == flushingQueue);
     }
     return(true);
 }
Example #9
0
 protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInt32 index, int?[] ids, CountdownEvent latch)
 {
     this.Queue = queue;
     this.Index = index;
     this.Ids   = ids;
     this.Slice = queue.NewSlice();
     Deletes    = new BufferedUpdates();
     this.Latch = latch;
 }
Example #10
0
        public virtual void TestUpdateDelteSlices()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
            int size = 200 + Random.Next(500) * RANDOM_MULTIPLIER;

            int?[] ids = new int?[size];
            for (int i = 0; i < ids.Length; i++)
            {
                ids[i] = Random.Next();
            }
            DeleteSlice     slice1       = queue.NewSlice();
            DeleteSlice     slice2       = queue.NewSlice();
            BufferedUpdates bd1          = new BufferedUpdates();
            BufferedUpdates bd2          = new BufferedUpdates();
            int             last1        = 0;
            int             last2        = 0;
            ISet <Term>     uniqueValues = new JCG.HashSet <Term>();

            for (int j = 0; j < ids.Length; j++)
            {
                int?i = ids[j];
                // create an array here since we compare identity below against tailItem
                Term[] term = new Term[] { new Term("id", i.ToString()) };
                uniqueValues.Add(term[0]);
                queue.AddDelete(term);
                if (Random.Next(20) == 0 || j == ids.Length - 1)
                {
                    queue.UpdateSlice(slice1);
                    Assert.IsTrue(slice1.IsTailItem(term));
                    slice1.Apply(bd1, j);
                    AssertAllBetween(last1, j, bd1, ids);
                    last1 = j + 1;
                }
                if (Random.Next(10) == 5 || j == ids.Length - 1)
                {
                    queue.UpdateSlice(slice2);
                    Assert.IsTrue(slice2.IsTailItem(term));
                    slice2.Apply(bd2, j);
                    AssertAllBetween(last2, j, bd2, ids);
                    last2 = j + 1;
                }
                Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes);
            }
            assertEquals(uniqueValues, new JCG.HashSet <Term>(bd1.terms.Keys));
            assertEquals(uniqueValues, new JCG.HashSet <Term>(bd2.terms.Keys));
            var frozenSet = new JCG.HashSet <Term>();

            foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
            {
                BytesRef bytesRef = new BytesRef();
                bytesRef.CopyBytes(t.Bytes);
                frozenSet.Add(new Term(t.Field, bytesRef));
            }
            assertEquals(uniqueValues, frozenSet);
            Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
        }
Example #11
0
 internal bool UpdateNumericDocValue(Term term, string field, long?value)
 {
     lock (this)
     {
         DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
         deleteQueue.AddNumericUpdate(new NumericDocValuesUpdate(term, field, value));
         flushControl.DoOnDelete();
         return(ApplyAllDeletes(deleteQueue));
     }
 }
Example #12
0
 internal bool UpdateBinaryDocValue(Term term, string field, BytesRef value)
 {
     lock (this)
     {
         DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
         deleteQueue.AddBinaryUpdate(new BinaryDocValuesUpdate(term, field, value));
         flushControl.DoOnDelete();
         return(ApplyAllDeletes(deleteQueue));
     }
 }
Example #13
0
 internal bool DeleteQueries(params Query[] queries)
 {
     lock (this)
     {
         // TODO why is this synchronized?
         DocumentsWriterDeleteQueue deleteQueue = this.DeleteQueue;
         deleteQueue.AddDelete(queries);
         FlushControl.DoOnDelete();
         return(ApplyAllDeletes(deleteQueue));
     }
 }
Example #14
0
 // TODO: we could check w/ FreqProxTermsWriter: if the
 // term doesn't exist, don't bother buffering into the
 // per-DWPT map (but still must go into the global map)
 internal bool DeleteTerms(params Term[] terms)
 {
     lock (this)
     {
         // TODO why is this synchronized?
         DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
         deleteQueue.AddDelete(terms);
         flushControl.DoOnDelete();
         return(ApplyAllDeletes(deleteQueue));
     }
 }
 public virtual void TestUpdateDelteSlices()
 {
     DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
     int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
     int?[] ids = new int?[size];
     for (int i = 0; i < ids.Length; i++)
     {
         ids[i] = Random().Next();
     }
     DeleteSlice slice1 = queue.NewSlice();
     DeleteSlice slice2 = queue.NewSlice();
     BufferedUpdates bd1 = new BufferedUpdates();
     BufferedUpdates bd2 = new BufferedUpdates();
     int last1 = 0;
     int last2 = 0;
     HashSet<Term> uniqueValues = new HashSet<Term>();
     for (int j = 0; j < ids.Length; j++)
     {
         int? i = ids[j];
         // create an array here since we compare identity below against tailItem
         Term[] term = new Term[] { new Term("id", i.ToString()) };
         uniqueValues.Add(term[0]);
         queue.AddDelete(term);
         if (Random().Next(20) == 0 || j == ids.Length - 1)
         {
             queue.UpdateSlice(slice1);
             Assert.IsTrue(slice1.IsTailItem(term));
             slice1.Apply(bd1, j);
             AssertAllBetween(last1, j, bd1, ids);
             last1 = j + 1;
         }
         if (Random().Next(10) == 5 || j == ids.Length - 1)
         {
             queue.UpdateSlice(slice2);
             Assert.IsTrue(slice2.IsTailItem(term));
             slice2.Apply(bd2, j);
             AssertAllBetween(last2, j, bd2, ids);
             last2 = j + 1;
         }
         Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes());
     }
     Assert.AreEqual(uniqueValues, bd1.Terms_Nunit().Keys);
     Assert.AreEqual(uniqueValues, bd2.Terms_Nunit().Keys);
     HashSet<Term> frozenSet = new HashSet<Term>();
     foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
     {
         BytesRef bytesRef = new BytesRef();
         bytesRef.CopyBytes(t.Bytes());
         frozenSet.Add(new Term(t.Field(), bytesRef));
     }
     Assert.AreEqual(uniqueValues, frozenSet);
     Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
 }
Example #16
0
 // for asserts
 private bool SetFlushingDeleteQueue(DocumentsWriterDeleteQueue session)
 {
     UninterruptableMonitor.Enter(this);
     try
     {
         currentFullFlushDelQueue = session;
         return(true);
     }
     finally
     {
         UninterruptableMonitor.Exit(this);
     }
 }
Example #17
0
 private bool ApplyAllDeletes(DocumentsWriterDeleteQueue deleteQueue)
 {
     if (flushControl.GetAndResetApplyAllDeletes())
     {
         if (deleteQueue != null && !flushControl.IsFullFlush)
         {
             ticketQueue.AddDeletes(deleteQueue);
         }
         PutEvent(ApplyDeletesEvent.INSTANCE); // apply deletes event forces a purge
         return(true);
     }
     return(false);
 }
Example #18
0
        public virtual void TestStressDeleteQueue()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
            ISet <Term> uniqueValues         = new JCG.HashSet <Term>();
            int         size = 10000 + Random.Next(500) * RANDOM_MULTIPLIER;

            int?[] ids = new int?[size];
            for (int i = 0; i < ids.Length; i++)
            {
                ids[i] = Random.Next();
                uniqueValues.Add(new Term("id", ids[i].ToString()));
            }
            CountdownEvent latch      = new CountdownEvent(1);
            AtomicInt32    index      = new AtomicInt32(0);
            int            numThreads = 2 + Random.Next(5);

            UpdateThread[] threads = new UpdateThread[numThreads];
            for (int i = 0; i < threads.Length; i++)
            {
                threads[i] = new UpdateThread(queue, index, ids, latch);
                threads[i].Start();
            }
            latch.Signal();
            for (int i = 0; i < threads.Length; i++)
            {
                threads[i].Join();
            }

            foreach (UpdateThread updateThread in threads)
            {
                DeleteSlice slice = updateThread.Slice;
                queue.UpdateSlice(slice);
                BufferedUpdates deletes = updateThread.Deletes;
                slice.Apply(deletes, BufferedUpdates.MAX_INT32);
                assertEquals(uniqueValues, new JCG.HashSet <Term>(deletes.terms.Keys));
            }
            queue.TryApplyGlobalSlice();
            ISet <Term> frozenSet = new JCG.HashSet <Term>();

            foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
            {
                BytesRef bytesRef = new BytesRef();
                bytesRef.CopyBytes(t.Bytes);
                frozenSet.Add(new Term(t.Field, bytesRef));
            }
            Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
            Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
            assertEquals(uniqueValues, frozenSet);
        }
Example #19
0
 internal bool UpdateBinaryDocValue(Term term, string field, BytesRef value)
 {
     UninterruptableMonitor.Enter(this);
     try
     {
         DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
         deleteQueue.AddBinaryUpdate(new BinaryDocValuesUpdate(term, field, value));
         flushControl.DoOnDelete();
         return(ApplyAllDeletes(deleteQueue));
     }
     finally
     {
         UninterruptableMonitor.Exit(this);
     }
 }
Example #20
0
 // TODO: we could check w/ FreqProxTermsWriter: if the
 // term doesn't exist, don't bother buffering into the
 // per-DWPT map (but still must go into the global map)
 internal bool DeleteTerms(params Term[] terms)
 {
     UninterruptableMonitor.Enter(this);
     try
     {
         // TODO why is this synchronized?
         DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
         deleteQueue.AddDelete(terms);
         flushControl.DoOnDelete();
         return(ApplyAllDeletes(deleteQueue));
     }
     finally
     {
         UninterruptableMonitor.Exit(this);
     }
 }
Example #21
0
        private bool AssertActiveDeleteQueue(DocumentsWriterDeleteQueue queue)
        {
            int limit = perThreadPool.NumThreadStatesActive;

            for (int i = 0; i < limit; i++)
            {
                ThreadState next = perThreadPool.GetThreadState(i);
                next.@Lock();
                try
                {
                    Debug.Assert(!next.IsInitialized || next.dwpt.deleteQueue == queue, "isInitialized: " + next.IsInitialized + " numDocs: " + (next.IsInitialized ? next.dwpt.NumDocsInRAM : 0));
                }
                finally
                {
                    next.Unlock();
                }
            }
            return(true);
        }
Example #22
0
        /// <summary>
        /// Prunes the blockedQueue by removing all DWPT that are associated with the given flush queue.
        /// </summary>
        private void PruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue)
        {
            var node = blockedFlushes.First;

            while (node != null)
            {
                var          nextNode     = node.Next;
                BlockedFlush blockedFlush = node.Value;
                if (blockedFlush.Dwpt.deleteQueue == flushingQueue)
                {
                    blockedFlushes.Remove(node);
                    Debug.Assert(!flushingWriters.ContainsKey(blockedFlush.Dwpt), "DWPT is already flushing");
                    // Record the flushing DWPT to reduce flushBytes in doAfterFlush
                    flushingWriters[blockedFlush.Dwpt] = Convert.ToInt64(blockedFlush.Bytes);
                    // don't decr pending here - its already done when DWPT is blocked
                    flushQueue.Enqueue(blockedFlush.Dwpt);
                }
                node = nextNode;
            }
        }
 internal virtual void AddDeletes(DocumentsWriterDeleteQueue deleteQueue)
 {
     lock (this)
     {
         IncTickets(); // first inc the ticket count - freeze opens
         // a window for #anyChanges to fail
         bool success = false;
         try
         {
             Queue.AddLast(new GlobalDeletesTicket(deleteQueue.FreezeGlobalBuffer(null)));
             success = true;
         }
         finally
         {
             if (!success)
             {
                 DecTickets();
             }
         }
     }
 }
 internal virtual void AddDeletes(DocumentsWriterDeleteQueue deleteQueue)
 {
     lock (this)
     {
         IncTickets(); // first inc the ticket count - freeze opens
         // a window for #anyChanges to fail
         bool success = false;
         try
         {
             Queue.AddLast(new GlobalDeletesTicket(deleteQueue.FreezeGlobalBuffer(null)));
             success = true;
         }
         finally
         {
             if (!success)
             {
                 DecTickets();
             }
         }
     }
 }
 /// <summary>
 /// Prunes the blockedQueue by removing all DWPT that are associated with the given flush queue.
 /// </summary>
 private void PruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue)
 {
     lock (this)
     {
         IEnumerator <BlockedFlush> iterator = BlockedFlushes.GetEnumerator();
         while (iterator.MoveNext())
         {
             BlockedFlush blockedFlush = iterator.Current;
             if (blockedFlush.Dwpt.DeleteQueue == flushingQueue)
             {
                 //LUCENE TODO: Move to try finally
                 BlockedFlushes.Remove(blockedFlush);
                 Debug.Assert(!FlushingWriters.ContainsKey(blockedFlush.Dwpt), "DWPT is already flushing");
                 // Record the flushing DWPT to reduce flushBytes in doAfterFlush
                 FlushingWriters[blockedFlush.Dwpt] = Convert.ToInt64(blockedFlush.Bytes);
                 // don't decr pending here - its already done when DWPT is blocked
                 FlushQueue.Enqueue(blockedFlush.Dwpt);
             }
         }
     }
 }
Example #26
0
        public virtual void TestPartiallyAppliedGlobalSlice()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();

            System.Reflection.FieldInfo field = typeof(DocumentsWriterDeleteQueue).GetField("globalBufferLock",
                                                                                            BindingFlags.NonPublic | BindingFlags.GetField | BindingFlags.Instance);
            ReentrantLock @lock = (ReentrantLock)field.GetValue(queue);

            @lock.Lock();
            var t = new ThreadAnonymousInnerClassHelper(this, queue);

            t.Start();
            t.Join();
            @lock.Unlock();
            Assert.IsTrue(queue.AnyChanges(), "changes in del queue but not in slice yet");
            queue.TryApplyGlobalSlice();
            Assert.IsTrue(queue.AnyChanges(), "changes in global buffer");
            FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);

            Assert.IsTrue(freezeGlobalBuffer.Any());
            Assert.AreEqual(1, freezeGlobalBuffer.termCount);
            Assert.IsFalse(queue.AnyChanges(), "all changes applied");
        }
Example #27
0
 // for asserts
 private bool SetFlushingDeleteQueue(DocumentsWriterDeleteQueue session)
 {
     lock (this)
     {
         CurrentFullFlushDelQueue = session;
         return true;
     }
 }
Example #28
0
        internal void MarkForFullFlush()
        {
            DocumentsWriterDeleteQueue flushingQueue;

            lock (this)
            {
                Debug.Assert(!fullFlush, "called DWFC#markForFullFlush() while full flush is still running");
                Debug.Assert(fullFlushBuffer.Count == 0, "full flush buffer should be empty: " + fullFlushBuffer);
                fullFlush     = true;
                flushingQueue = documentsWriter.deleteQueue;
                // Set a new delete queue - all subsequent DWPT will use this queue until
                // we do another full flush
                DocumentsWriterDeleteQueue newQueue = new DocumentsWriterDeleteQueue(flushingQueue.generation + 1);
                documentsWriter.deleteQueue = newQueue;
            }
            int limit = perThreadPool.NumThreadStatesActive;

            for (int i = 0; i < limit; i++)
            {
                ThreadState next = perThreadPool.GetThreadState(i);
                next.@Lock();
                try
                {
                    if (!next.IsInitialized)
                    {
                        if (closed && next.IsActive)
                        {
                            perThreadPool.DeactivateThreadState(next);
                        }
                        continue;
                    }
                    Debug.Assert(next.dwpt.deleteQueue == flushingQueue || next.dwpt.deleteQueue == documentsWriter.deleteQueue, " flushingQueue: " + flushingQueue + " currentqueue: " + documentsWriter.deleteQueue + " perThread queue: " + next.dwpt.deleteQueue + " numDocsInRam: " + next.dwpt.NumDocsInRAM);
                    if (next.dwpt.deleteQueue != flushingQueue)
                    {
                        // this one is already a new DWPT
                        continue;
                    }
                    AddFlushableState(next);
                }
                finally
                {
                    next.Unlock();
                }
            }
            lock (this)
            {
                /* make sure we move all DWPT that are where concurrently marked as
                 * pending and moved to blocked are moved over to the flushQueue. There is
                 * a chance that this happens since we marking DWPT for full flush without
                 * blocking indexing.*/
                PruneBlockedQueue(flushingQueue);
                Debug.Assert(AssertBlockedFlushes(documentsWriter.deleteQueue));
                //FlushQueue.AddAll(FullFlushBuffer);
                foreach (var dwpt in fullFlushBuffer)
                {
                    flushQueue.Enqueue(dwpt);
                }
                fullFlushBuffer.Clear();
                UpdateStallState();
            }
            Debug.Assert(AssertActiveDeleteQueue(documentsWriter.deleteQueue));
        }
 public virtual void TestPartiallyAppliedGlobalSlice()
 {
     DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
     System.Reflection.FieldInfo field = typeof(DocumentsWriterDeleteQueue).GetField("GlobalBufferLock", 
         BindingFlags.NonPublic | BindingFlags.GetField | BindingFlags.Instance);
     ReentrantLock @lock = (ReentrantLock)field.GetValue(queue);
     @lock.Lock();
     ThreadClass t = new ThreadAnonymousInnerClassHelper(this, queue);
     t.Start();
     t.Join();
     @lock.Unlock();
     Assert.IsTrue(queue.AnyChanges(), "changes in del queue but not in slice yet");
     queue.TryApplyGlobalSlice();
     Assert.IsTrue(queue.AnyChanges(), "changes in global buffer");
     FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
     Assert.IsTrue(freezeGlobalBuffer.Any());
     Assert.AreEqual(1, freezeGlobalBuffer.TermCount);
     Assert.IsFalse(queue.AnyChanges(), "all changes applied");
 }
        public virtual void TestStressDeleteQueue()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
            HashSet<Term> uniqueValues = new HashSet<Term>();
            int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER;
            int?[] ids = new int?[size];
            for (int i = 0; i < ids.Length; i++)
            {
                ids[i] = Random().Next();
                uniqueValues.Add(new Term("id", ids[i].ToString()));
            }
            CountDownLatch latch = new CountDownLatch(1);
            AtomicInteger index = new AtomicInteger(0);
            int numThreads = 2 + Random().Next(5);
            UpdateThread[] threads = new UpdateThread[numThreads];
            for (int i = 0; i < threads.Length; i++)
            {
                threads[i] = new UpdateThread(queue, index, ids, latch);
                threads[i].Start();
            }
            latch.countDown();
            for (int i = 0; i < threads.Length; i++)
            {
                threads[i].Join();
            }

            foreach (UpdateThread updateThread in threads)
            {
                DeleteSlice slice = updateThread.Slice;
                queue.UpdateSlice(slice);
                BufferedUpdates deletes = updateThread.Deletes;
                slice.Apply(deletes, BufferedUpdates.MAX_INT);
                Assert.AreEqual(uniqueValues, deletes.Terms_Nunit().Keys);
            }
            queue.TryApplyGlobalSlice();
            HashSet<Term> frozenSet = new HashSet<Term>();
            foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
            {
                BytesRef bytesRef = new BytesRef();
                bytesRef.CopyBytes(t.Bytes());
                frozenSet.Add(new Term(t.Field(), bytesRef));
            }
            Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
            Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
            Assert.AreEqual(uniqueValues, frozenSet);
        }
Example #31
0
 private bool ApplyAllDeletes(DocumentsWriterDeleteQueue deleteQueue)
 {
     if (FlushControl.AndResetApplyAllDeletes)
     {
         if (deleteQueue != null && !FlushControl.FullFlush)
         {
             TicketQueue.AddDeletes(deleteQueue);
         }
         PutEvent(ApplyDeletesEvent.INSTANCE); // apply deletes event forces a purge
         return true;
     }
     return false;
 }
 public ThreadAnonymousInnerClassHelper(TestDocumentsWriterDeleteQueue outerInstance, DocumentsWriterDeleteQueue queue)
 {
     this.OuterInstance = outerInstance;
     this.Queue = queue;
 }
        public DocumentsWriterPerThread(string segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue, FieldInfos.Builder fieldInfos)
        {
            this.DirectoryOrig = directory;
            this.Directory = new TrackingDirectoryWrapper(directory);
            this.FieldInfos = fieldInfos;
            this.IndexWriterConfig = indexWriterConfig;
            this.InfoStream = infoStream;
            this.Codec = indexWriterConfig.Codec;
            this.docState = new DocState(this, infoStream);
            this.docState.Similarity = indexWriterConfig.Similarity;
            bytesUsed = Counter.NewCounter();
            ByteBlockAllocator = new DirectTrackingAllocator(bytesUsed);
            PendingUpdates = new BufferedUpdates();
            intBlockAllocator = new IntBlockAllocator(bytesUsed);
            this.DeleteQueue = deleteQueue;
            Debug.Assert(numDocsInRAM == 0, "num docs " + numDocsInRAM);
            PendingUpdates.Clear();
            DeleteSlice = deleteQueue.NewSlice();

            SegmentInfo_Renamed = new SegmentInfo(DirectoryOrig, Constants.LUCENE_MAIN_VERSION, segmentName, -1, false, Codec, null);
            Debug.Assert(numDocsInRAM == 0);
            if (INFO_VERBOSE && infoStream.IsEnabled("DWPT"))
            {
                infoStream.Message("DWPT", Thread.CurrentThread.Name + " init seg=" + segmentName + " delQueue=" + deleteQueue);
            }
            // this should be the last call in the ctor
            // it really sucks that we need to pull this within the ctor and pass this ref to the chain!
            Consumer = indexWriterConfig.IndexingChain.GetChain(this);
        }
Example #34
0
 public ThreadAnonymousInnerClassHelper(TestDocumentsWriterDeleteQueue outerInstance, DocumentsWriterDeleteQueue queue)
 {
     this.OuterInstance = outerInstance;
     this.Queue         = queue;
 }
Example #35
0
 public ThreadAnonymousClass(TestDocumentsWriterDeleteQueue outerInstance, DocumentsWriterDeleteQueue queue)
 {
     this.outerInstance = outerInstance;
     this.queue         = queue;
 }
 protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInteger index, int?[] ids, CountDownLatch latch)
 {
     this.Queue = queue;
     this.Index = index;
     this.Ids = ids;
     this.Slice = queue.NewSlice();
     Deletes = new BufferedUpdates();
     this.Latch = latch;
 }