public override void OnDelete(DocumentsWriterFlushControl control, ThreadState state)
 {
     if (FlushOnDeleteTerms())
     {
         // Flush this state by num del terms
         int maxBufferedDeleteTerms = IWConfig.MaxBufferedDeleteTerms;
         if (control.NumGlobalTermDeletes >= maxBufferedDeleteTerms)
         {
             control.SetApplyAllDeletes();
         }
     }
     if ((FlushOnRAM() && control.DeleteBytesUsed > (1024 * 1024 * IWConfig.RAMBufferSizeMB)))
     {
         control.SetApplyAllDeletes();
         if (InfoStream.IsEnabled("FP"))
         {
             InfoStream.Message("FP", "force apply deletes bytesUsed=" + control.DeleteBytesUsed + " vs ramBuffer=" + (1024 * 1024 * IWConfig.RAMBufferSizeMB));
         }
     }
 }
 public override void OnInsert(DocumentsWriterFlushControl control, ThreadState state)
 {
     if (FlushOnDocCount() && state.Dwpt.NumDocsInRAM >= IWConfig.MaxBufferedDocs)
     {
         // Flush this state by num docs
         control.FlushPending = state;
     } // flush by RAM
     else if (FlushOnRAM())
     {
         long limit = (long)(IWConfig.RAMBufferSizeMB * 1024d * 1024d);
         long totalRam = control.ActiveBytes() + control.DeleteBytesUsed;
         if (totalRam >= limit)
         {
             if (InfoStream.IsEnabled("FP"))
             {
                 InfoStream.Message("FP", "flush: activeBytes=" + control.ActiveBytes() + " deleteBytes=" + control.DeleteBytesUsed + " vs limit=" + limit);
             }
             MarkLargestWriterPending(control, state, totalRam);
         }
     }
 }
Exemplo n.º 3
0
 /// <summary>
 /// Called for each document addition on the given <seealso cref="ThreadState"/>s
 /// <seealso cref="DocumentsWriterPerThread"/>.
 /// <p>
 /// Note: this method is synchronized by the given
 /// <seealso cref="DocumentsWriterFlushControl"/> and it is guaranteed that the calling
 /// thread holds the lock on the given <seealso cref="ThreadState"/>
 /// </summary>
 public abstract void OnInsert(DocumentsWriterFlushControl control, ThreadState state);
Exemplo n.º 4
0
 /// <summary>
 /// Returns the current most RAM consuming non-pending <seealso cref="ThreadState"/> with
 /// at least one indexed document.
 /// <p>
 /// this method will never return <code>null</code>
 /// </summary>
 protected internal virtual ThreadState FindLargestNonPendingWriter(DocumentsWriterFlushControl control, ThreadState perThreadState)
 {
     Debug.Assert(perThreadState.Dwpt.NumDocsInRAM > 0);
     long maxRamSoFar = perThreadState.BytesUsed;
     // the dwpt which needs to be flushed eventually
     ThreadState maxRamUsingThreadState = perThreadState;
     Debug.Assert(!perThreadState.FlushPending_Renamed, "DWPT should have flushed");
     IEnumerator<ThreadState> activePerThreadsIterator = control.AllActiveThreadStates();
     while (activePerThreadsIterator.MoveNext())
     {
         ThreadState next = activePerThreadsIterator.Current;
         if (!next.FlushPending_Renamed)
         {
             long nextRam = next.BytesUsed;
             if (nextRam > maxRamSoFar && next.Dwpt.NumDocsInRAM > 0)
             {
                 maxRamSoFar = nextRam;
                 maxRamUsingThreadState = next;
             }
         }
     }
     Debug.Assert(AssertMessage("set largest ram consuming thread pending on lower watermark"));
     return maxRamUsingThreadState;
 }
Exemplo n.º 5
0
 /// <summary>
 /// Called for each document update on the given <seealso cref="ThreadState"/>'s
 /// <seealso cref="DocumentsWriterPerThread"/>.
 /// <p>
 /// Note: this method is called  synchronized on the given
 /// <seealso cref="DocumentsWriterFlushControl"/> and it is guaranteed that the calling
 /// thread holds the lock on the given <seealso cref="ThreadState"/>
 /// </summary>
 public virtual void OnUpdate(DocumentsWriterFlushControl control, ThreadState state)
 {
     OnInsert(control, state);
     OnDelete(control, state);
 }
 internal static void FindPending(DocumentsWriterFlushControl flushControl, List<ThreadState> pending, List<ThreadState> notPending)
 {
     IEnumerator<ThreadState> allActiveThreads = flushControl.AllActiveThreadStates();
     while (allActiveThreads.MoveNext())
     {
         ThreadState next = allActiveThreads.Current;
         if (next.FlushPending)
         {
             pending.Add(next);
         }
         else
         {
             notPending.Add(next);
         }
     }
 }
            public override void OnInsert(DocumentsWriterFlushControl control, ThreadState state)
            {
                List<ThreadState> pending = new List<ThreadState>();
                List<ThreadState> notPending = new List<ThreadState>();
                FindPending(control, pending, notPending);
                bool flushCurrent = state.FlushPending;
                long activeBytes = control.ActiveBytes();
                ThreadState toFlush;
                if (state.FlushPending)
                {
                    toFlush = state;
                }
                else if (FlushOnDocCount() && state.DocumentsWriterPerThread.NumDocsInRAM >= IWConfig.MaxBufferedDocs)
                {
                    toFlush = state;
                }
                else if (FlushOnRAM() && activeBytes >= (long)(IWConfig.RAMBufferSizeMB * 1024.0 * 1024.0))
                {
                    toFlush = FindLargestNonPendingWriter(control, state);
                    Assert.IsFalse(toFlush.FlushPending);
                }
                else
                {
                    toFlush = null;
                }
                base.OnInsert(control, state);
                if (toFlush != null)
                {
                    if (flushCurrent)
                    {
                        Assert.IsTrue(pending.Remove(toFlush));
                    }
                    else
                    {
                        Assert.IsTrue(notPending.Remove(toFlush));
                    }
                    Assert.IsTrue(toFlush.FlushPending);
                    HasMarkedPending = true;
                }
                else
                {
                    PeakBytesWithoutFlush = Math.Max(activeBytes, PeakBytesWithoutFlush);
                    PeakDocCountWithoutFlush = Math.Max(state.DocumentsWriterPerThread.NumDocsInRAM, PeakDocCountWithoutFlush);
                }

                foreach (ThreadState threadState in notPending)
                {
                    Assert.IsFalse(threadState.FlushPending);
                }
            }
            public override void OnDelete(DocumentsWriterFlushControl control, ThreadState state)
            {
                List<ThreadState> pending = new List<ThreadState>();
                List<ThreadState> notPending = new List<ThreadState>();
                FindPending(control, pending, notPending);
                bool flushCurrent = state.FlushPending;
                ThreadState toFlush;
                if (state.FlushPending)
                {
                    toFlush = state;
                }
                else if (FlushOnDeleteTerms() && state.DocumentsWriterPerThread.NumDeleteTerms() >= IWConfig.MaxBufferedDeleteTerms)
                {
                    toFlush = state;
                }
                else
                {
                    toFlush = null;
                }
                base.OnDelete(control, state);
                if (toFlush != null)
                {
                    if (flushCurrent)
                    {
                        Assert.IsTrue(pending.Remove(toFlush));
                    }
                    else
                    {
                        Assert.IsTrue(notPending.Remove(toFlush));
                    }
                    Assert.IsTrue(toFlush.FlushPending);
                    HasMarkedPending = true;
                }

                foreach (ThreadState threadState in notPending)
                {
                    Assert.IsFalse(threadState.FlushPending);
                }
            }
 protected internal virtual void AssertActiveBytesAfter(DocumentsWriterFlushControl flushControl)
 {
     IEnumerator<ThreadState> allActiveThreads = flushControl.AllActiveThreadStates();
     long bytesUsed = 0;
     while (allActiveThreads.MoveNext())
     {
         ThreadState next = allActiveThreads.Current;
         if (next.DocumentsWriterPerThread != null)
         {
             bytesUsed += next.DocumentsWriterPerThread.BytesUsed();
         }
     }
     Assert.AreEqual(bytesUsed, flushControl.ActiveBytes());
 }
Exemplo n.º 10
0
 internal DocumentsWriter(IndexWriter writer, LiveIndexWriterConfig config, Directory directory)
 {
     this.Directory = directory;
     this.LIWConfig = config;
     this.InfoStream = config.InfoStream;
     this.PerThreadPool = config.IndexerThreadPool;
     FlushPolicy = config.FlushPolicy;
     this.Writer = writer;
     this.Events = new ConcurrentQueue<Event>();
     FlushControl = new DocumentsWriterFlushControl(this, config, writer.BufferedUpdatesStream);
 }
Exemplo n.º 11
0
 /// <summary>
 /// Marks the most ram consuming active <seealso cref="DocumentsWriterPerThread"/> flush
 /// pending
 /// </summary>
 protected internal virtual void MarkLargestWriterPending(DocumentsWriterFlushControl control, ThreadState perThreadState, long currentBytesPerThread)
 {
     control.FlushPending = FindLargestNonPendingWriter(control, perThreadState);
 }