FreezeGlobalBuffer() public method

public FreezeGlobalBuffer ( DeleteSlice callerSlice ) : FrozenBufferedUpdates
callerSlice DeleteSlice
return FrozenBufferedUpdates
 public virtual void TestAnyChanges()
 {
     DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
     int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
     int termsSinceFreeze = 0;
     int queriesSinceFreeze = 0;
     for (int i = 0; i < size; i++)
     {
         Term term = new Term("id", "" + i);
         if (Random().Next(10) == 0)
         {
             queue.AddDelete(new TermQuery(term));
             queriesSinceFreeze++;
         }
         else
         {
             queue.AddDelete(term);
             termsSinceFreeze++;
         }
         Assert.IsTrue(queue.AnyChanges());
         if (Random().Next(5) == 0)
         {
             FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
             Assert.AreEqual(termsSinceFreeze, freezeGlobalBuffer.TermCount);
             Assert.AreEqual(queriesSinceFreeze, ((Query[])freezeGlobalBuffer.Queries_Nunit()).Length);
             queriesSinceFreeze = 0;
             termsSinceFreeze = 0;
             Assert.IsFalse(queue.AnyChanges());
         }
     }
 }
Exemplo n.º 2
0
        public virtual void TestAnyChanges()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
            int size               = 200 + Random.Next(500) * RANDOM_MULTIPLIER;
            int termsSinceFreeze   = 0;
            int queriesSinceFreeze = 0;

            for (int i = 0; i < size; i++)
            {
                Term term = new Term("id", "" + i);
                if (Random.Next(10) == 0)
                {
                    queue.AddDelete(new TermQuery(term));
                    queriesSinceFreeze++;
                }
                else
                {
                    queue.AddDelete(term);
                    termsSinceFreeze++;
                }
                Assert.IsTrue(queue.AnyChanges());
                if (Random.Next(5) == 0)
                {
                    FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
                    Assert.AreEqual(termsSinceFreeze, freezeGlobalBuffer.termCount);
                    Assert.AreEqual(queriesSinceFreeze, ((Query[])freezeGlobalBuffer.queries.Clone()).Length);
                    queriesSinceFreeze = 0;
                    termsSinceFreeze   = 0;
                    Assert.IsFalse(queue.AnyChanges());
                }
            }
        }
Exemplo n.º 3
0
 internal virtual void AddDeletes(DocumentsWriterDeleteQueue deleteQueue)
 {
     UninterruptableMonitor.Enter(this);
     try
     {
         IncTickets(); // first inc the ticket count - freeze opens
         // a window for #anyChanges to fail
         bool success = false;
         try
         {
             queue.Enqueue(new GlobalDeletesTicket(deleteQueue.FreezeGlobalBuffer(null)));
             success = true;
         }
         finally
         {
             if (!success)
             {
                 DecTickets();
             }
         }
     }
     finally
     {
         UninterruptableMonitor.Exit(this);
     }
 }
Exemplo n.º 4
0
        public virtual void TestUpdateDelteSlices()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
            int size = 200 + Random.Next(500) * RANDOM_MULTIPLIER;

            int?[] ids = new int?[size];
            for (int i = 0; i < ids.Length; i++)
            {
                ids[i] = Random.Next();
            }
            DeleteSlice     slice1       = queue.NewSlice();
            DeleteSlice     slice2       = queue.NewSlice();
            BufferedUpdates bd1          = new BufferedUpdates();
            BufferedUpdates bd2          = new BufferedUpdates();
            int             last1        = 0;
            int             last2        = 0;
            ISet <Term>     uniqueValues = new JCG.HashSet <Term>();

            for (int j = 0; j < ids.Length; j++)
            {
                int?i = ids[j];
                // create an array here since we compare identity below against tailItem
                Term[] term = new Term[] { new Term("id", i.ToString()) };
                uniqueValues.Add(term[0]);
                queue.AddDelete(term);
                if (Random.Next(20) == 0 || j == ids.Length - 1)
                {
                    queue.UpdateSlice(slice1);
                    Assert.IsTrue(slice1.IsTailItem(term));
                    slice1.Apply(bd1, j);
                    AssertAllBetween(last1, j, bd1, ids);
                    last1 = j + 1;
                }
                if (Random.Next(10) == 5 || j == ids.Length - 1)
                {
                    queue.UpdateSlice(slice2);
                    Assert.IsTrue(slice2.IsTailItem(term));
                    slice2.Apply(bd2, j);
                    AssertAllBetween(last2, j, bd2, ids);
                    last2 = j + 1;
                }
                Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes);
            }
            assertEquals(uniqueValues, new JCG.HashSet <Term>(bd1.terms.Keys));
            assertEquals(uniqueValues, new JCG.HashSet <Term>(bd2.terms.Keys));
            var frozenSet = new JCG.HashSet <Term>();

            foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
            {
                BytesRef bytesRef = new BytesRef();
                bytesRef.CopyBytes(t.Bytes);
                frozenSet.Add(new Term(t.Field, bytesRef));
            }
            assertEquals(uniqueValues, frozenSet);
            Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
        }
 public virtual void TestUpdateDelteSlices()
 {
     DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
     int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
     int?[] ids = new int?[size];
     for (int i = 0; i < ids.Length; i++)
     {
         ids[i] = Random().Next();
     }
     DeleteSlice slice1 = queue.NewSlice();
     DeleteSlice slice2 = queue.NewSlice();
     BufferedUpdates bd1 = new BufferedUpdates();
     BufferedUpdates bd2 = new BufferedUpdates();
     int last1 = 0;
     int last2 = 0;
     HashSet<Term> uniqueValues = new HashSet<Term>();
     for (int j = 0; j < ids.Length; j++)
     {
         int? i = ids[j];
         // create an array here since we compare identity below against tailItem
         Term[] term = new Term[] { new Term("id", i.ToString()) };
         uniqueValues.Add(term[0]);
         queue.AddDelete(term);
         if (Random().Next(20) == 0 || j == ids.Length - 1)
         {
             queue.UpdateSlice(slice1);
             Assert.IsTrue(slice1.IsTailItem(term));
             slice1.Apply(bd1, j);
             AssertAllBetween(last1, j, bd1, ids);
             last1 = j + 1;
         }
         if (Random().Next(10) == 5 || j == ids.Length - 1)
         {
             queue.UpdateSlice(slice2);
             Assert.IsTrue(slice2.IsTailItem(term));
             slice2.Apply(bd2, j);
             AssertAllBetween(last2, j, bd2, ids);
             last2 = j + 1;
         }
         Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes());
     }
     Assert.AreEqual(uniqueValues, bd1.Terms_Nunit().Keys);
     Assert.AreEqual(uniqueValues, bd2.Terms_Nunit().Keys);
     HashSet<Term> frozenSet = new HashSet<Term>();
     foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
     {
         BytesRef bytesRef = new BytesRef();
         bytesRef.CopyBytes(t.Bytes());
         frozenSet.Add(new Term(t.Field(), bytesRef));
     }
     Assert.AreEqual(uniqueValues, frozenSet);
     Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
 }
Exemplo n.º 6
0
        public virtual void TestStressDeleteQueue()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
            ISet <Term> uniqueValues         = new JCG.HashSet <Term>();
            int         size = 10000 + Random.Next(500) * RANDOM_MULTIPLIER;

            int?[] ids = new int?[size];
            for (int i = 0; i < ids.Length; i++)
            {
                ids[i] = Random.Next();
                uniqueValues.Add(new Term("id", ids[i].ToString()));
            }
            CountdownEvent latch      = new CountdownEvent(1);
            AtomicInt32    index      = new AtomicInt32(0);
            int            numThreads = 2 + Random.Next(5);

            UpdateThread[] threads = new UpdateThread[numThreads];
            for (int i = 0; i < threads.Length; i++)
            {
                threads[i] = new UpdateThread(queue, index, ids, latch);
                threads[i].Start();
            }
            latch.Signal();
            for (int i = 0; i < threads.Length; i++)
            {
                threads[i].Join();
            }

            foreach (UpdateThread updateThread in threads)
            {
                DeleteSlice slice = updateThread.Slice;
                queue.UpdateSlice(slice);
                BufferedUpdates deletes = updateThread.Deletes;
                slice.Apply(deletes, BufferedUpdates.MAX_INT32);
                assertEquals(uniqueValues, new JCG.HashSet <Term>(deletes.terms.Keys));
            }
            queue.TryApplyGlobalSlice();
            ISet <Term> frozenSet = new JCG.HashSet <Term>();

            foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
            {
                BytesRef bytesRef = new BytesRef();
                bytesRef.CopyBytes(t.Bytes);
                frozenSet.Add(new Term(t.Field, bytesRef));
            }
            Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
            Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
            assertEquals(uniqueValues, frozenSet);
        }
        public virtual int NumDocsInRAM => numDocsInRAM; // public for FlushPolicy

        /// <summary>
        /// Prepares this DWPT for flushing. this method will freeze and return the
        /// <see cref="DocumentsWriterDeleteQueue"/>s global buffer and apply all pending
        /// deletes to this DWPT.
        /// </summary>
        internal virtual FrozenBufferedUpdates PrepareFlush()
        {
            Debug.Assert(numDocsInRAM > 0);
            FrozenBufferedUpdates globalUpdates = deleteQueue.FreezeGlobalBuffer(deleteSlice);

            /* deleteSlice can possibly be null if we have hit non-aborting exceptions during indexing and never succeeded
             * adding a document. */
            if (deleteSlice != null)
            {
                // apply all deletes before we flush and release the delete slice
                deleteSlice.Apply(pendingUpdates, numDocsInRAM);
                Debug.Assert(deleteSlice.IsEmpty);
                deleteSlice.Reset();
            }
            return(globalUpdates);
        }
 internal virtual void AddDeletes(DocumentsWriterDeleteQueue deleteQueue)
 {
     lock (this)
     {
         IncTickets(); // first inc the ticket count - freeze opens
         // a window for #anyChanges to fail
         bool success = false;
         try
         {
             Queue.AddLast(new GlobalDeletesTicket(deleteQueue.FreezeGlobalBuffer(null)));
             success = true;
         }
         finally
         {
             if (!success)
             {
                 DecTickets();
             }
         }
     }
 }
Exemplo n.º 9
0
 internal virtual void AddDeletes(DocumentsWriterDeleteQueue deleteQueue)
 {
     lock (this)
     {
         IncTickets(); // first inc the ticket count - freeze opens
         // a window for #anyChanges to fail
         bool success = false;
         try
         {
             Queue.AddLast(new GlobalDeletesTicket(deleteQueue.FreezeGlobalBuffer(null)));
             success = true;
         }
         finally
         {
             if (!success)
             {
                 DecTickets();
             }
         }
     }
 }
Exemplo n.º 10
0
        public virtual void TestPartiallyAppliedGlobalSlice()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();

            System.Reflection.FieldInfo field = typeof(DocumentsWriterDeleteQueue).GetField("globalBufferLock",
                                                                                            BindingFlags.NonPublic | BindingFlags.GetField | BindingFlags.Instance);
            ReentrantLock @lock = (ReentrantLock)field.GetValue(queue);

            @lock.Lock();
            var t = new ThreadAnonymousInnerClassHelper(this, queue);

            t.Start();
            t.Join();
            @lock.Unlock();
            Assert.IsTrue(queue.AnyChanges(), "changes in del queue but not in slice yet");
            queue.TryApplyGlobalSlice();
            Assert.IsTrue(queue.AnyChanges(), "changes in global buffer");
            FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);

            Assert.IsTrue(freezeGlobalBuffer.Any());
            Assert.AreEqual(1, freezeGlobalBuffer.termCount);
            Assert.IsFalse(queue.AnyChanges(), "all changes applied");
        }
        public virtual void TestStressDeleteQueue()
        {
            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
            HashSet<Term> uniqueValues = new HashSet<Term>();
            int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER;
            int?[] ids = new int?[size];
            for (int i = 0; i < ids.Length; i++)
            {
                ids[i] = Random().Next();
                uniqueValues.Add(new Term("id", ids[i].ToString()));
            }
            CountDownLatch latch = new CountDownLatch(1);
            AtomicInteger index = new AtomicInteger(0);
            int numThreads = 2 + Random().Next(5);
            UpdateThread[] threads = new UpdateThread[numThreads];
            for (int i = 0; i < threads.Length; i++)
            {
                threads[i] = new UpdateThread(queue, index, ids, latch);
                threads[i].Start();
            }
            latch.countDown();
            for (int i = 0; i < threads.Length; i++)
            {
                threads[i].Join();
            }

            foreach (UpdateThread updateThread in threads)
            {
                DeleteSlice slice = updateThread.Slice;
                queue.UpdateSlice(slice);
                BufferedUpdates deletes = updateThread.Deletes;
                slice.Apply(deletes, BufferedUpdates.MAX_INT);
                Assert.AreEqual(uniqueValues, deletes.Terms_Nunit().Keys);
            }
            queue.TryApplyGlobalSlice();
            HashSet<Term> frozenSet = new HashSet<Term>();
            foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
            {
                BytesRef bytesRef = new BytesRef();
                bytesRef.CopyBytes(t.Bytes());
                frozenSet.Add(new Term(t.Field(), bytesRef));
            }
            Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
            Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
            Assert.AreEqual(uniqueValues, frozenSet);
        }
 public virtual void TestPartiallyAppliedGlobalSlice()
 {
     DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
     System.Reflection.FieldInfo field = typeof(DocumentsWriterDeleteQueue).GetField("GlobalBufferLock", 
         BindingFlags.NonPublic | BindingFlags.GetField | BindingFlags.Instance);
     ReentrantLock @lock = (ReentrantLock)field.GetValue(queue);
     @lock.Lock();
     ThreadClass t = new ThreadAnonymousInnerClassHelper(this, queue);
     t.Start();
     t.Join();
     @lock.Unlock();
     Assert.IsTrue(queue.AnyChanges(), "changes in del queue but not in slice yet");
     queue.TryApplyGlobalSlice();
     Assert.IsTrue(queue.AnyChanges(), "changes in global buffer");
     FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
     Assert.IsTrue(freezeGlobalBuffer.Any());
     Assert.AreEqual(1, freezeGlobalBuffer.TermCount);
     Assert.IsFalse(queue.AnyChanges(), "all changes applied");
 }