public override void OnDelete(DocumentsWriterFlushControl control, ThreadState state) { if (FlushOnDeleteTerms()) { // Flush this state by num del terms int maxBufferedDeleteTerms = IWConfig.MaxBufferedDeleteTerms; if (control.NumGlobalTermDeletes >= maxBufferedDeleteTerms) { control.SetApplyAllDeletes(); } } if ((FlushOnRAM() && control.DeleteBytesUsed > (1024 * 1024 * IWConfig.RAMBufferSizeMB))) { control.SetApplyAllDeletes(); if (InfoStream.IsEnabled("FP")) { InfoStream.Message("FP", "force apply deletes bytesUsed=" + control.DeleteBytesUsed + " vs ramBuffer=" + (1024 * 1024 * IWConfig.RAMBufferSizeMB)); } } }
public override void OnInsert(DocumentsWriterFlushControl control, ThreadState state) { if (FlushOnDocCount() && state.Dwpt.NumDocsInRAM >= IWConfig.MaxBufferedDocs) { // Flush this state by num docs control.FlushPending = state; } // flush by RAM else if (FlushOnRAM()) { long limit = (long)(IWConfig.RAMBufferSizeMB * 1024d * 1024d); long totalRam = control.ActiveBytes() + control.DeleteBytesUsed; if (totalRam >= limit) { if (InfoStream.IsEnabled("FP")) { InfoStream.Message("FP", "flush: activeBytes=" + control.ActiveBytes() + " deleteBytes=" + control.DeleteBytesUsed + " vs limit=" + limit); } MarkLargestWriterPending(control, state, totalRam); } } }
/// <summary> /// Called for each document addition on the given <seealso cref="ThreadState"/>s /// <seealso cref="DocumentsWriterPerThread"/>. /// <p> /// Note: this method is synchronized by the given /// <seealso cref="DocumentsWriterFlushControl"/> and it is guaranteed that the calling /// thread holds the lock on the given <seealso cref="ThreadState"/> /// </summary> public abstract void OnInsert(DocumentsWriterFlushControl control, ThreadState state);
/// <summary> /// Returns the current most RAM consuming non-pending <seealso cref="ThreadState"/> with /// at least one indexed document. /// <p> /// this method will never return <code>null</code> /// </summary> protected internal virtual ThreadState FindLargestNonPendingWriter(DocumentsWriterFlushControl control, ThreadState perThreadState) { Debug.Assert(perThreadState.Dwpt.NumDocsInRAM > 0); long maxRamSoFar = perThreadState.BytesUsed; // the dwpt which needs to be flushed eventually ThreadState maxRamUsingThreadState = perThreadState; Debug.Assert(!perThreadState.FlushPending_Renamed, "DWPT should have flushed"); IEnumerator<ThreadState> activePerThreadsIterator = control.AllActiveThreadStates(); while (activePerThreadsIterator.MoveNext()) { ThreadState next = activePerThreadsIterator.Current; if (!next.FlushPending_Renamed) { long nextRam = next.BytesUsed; if (nextRam > maxRamSoFar && next.Dwpt.NumDocsInRAM > 0) { maxRamSoFar = nextRam; maxRamUsingThreadState = next; } } } Debug.Assert(AssertMessage("set largest ram consuming thread pending on lower watermark")); return maxRamUsingThreadState; }
/// <summary> /// Called for each document update on the given <seealso cref="ThreadState"/>'s /// <seealso cref="DocumentsWriterPerThread"/>. /// <p> /// Note: this method is called synchronized on the given /// <seealso cref="DocumentsWriterFlushControl"/> and it is guaranteed that the calling /// thread holds the lock on the given <seealso cref="ThreadState"/> /// </summary> public virtual void OnUpdate(DocumentsWriterFlushControl control, ThreadState state) { OnInsert(control, state); OnDelete(control, state); }
internal static void FindPending(DocumentsWriterFlushControl flushControl, List<ThreadState> pending, List<ThreadState> notPending) { IEnumerator<ThreadState> allActiveThreads = flushControl.AllActiveThreadStates(); while (allActiveThreads.MoveNext()) { ThreadState next = allActiveThreads.Current; if (next.FlushPending) { pending.Add(next); } else { notPending.Add(next); } } }
public override void OnInsert(DocumentsWriterFlushControl control, ThreadState state) { List<ThreadState> pending = new List<ThreadState>(); List<ThreadState> notPending = new List<ThreadState>(); FindPending(control, pending, notPending); bool flushCurrent = state.FlushPending; long activeBytes = control.ActiveBytes(); ThreadState toFlush; if (state.FlushPending) { toFlush = state; } else if (FlushOnDocCount() && state.DocumentsWriterPerThread.NumDocsInRAM >= IWConfig.MaxBufferedDocs) { toFlush = state; } else if (FlushOnRAM() && activeBytes >= (long)(IWConfig.RAMBufferSizeMB * 1024.0 * 1024.0)) { toFlush = FindLargestNonPendingWriter(control, state); Assert.IsFalse(toFlush.FlushPending); } else { toFlush = null; } base.OnInsert(control, state); if (toFlush != null) { if (flushCurrent) { Assert.IsTrue(pending.Remove(toFlush)); } else { Assert.IsTrue(notPending.Remove(toFlush)); } Assert.IsTrue(toFlush.FlushPending); HasMarkedPending = true; } else { PeakBytesWithoutFlush = Math.Max(activeBytes, PeakBytesWithoutFlush); PeakDocCountWithoutFlush = Math.Max(state.DocumentsWriterPerThread.NumDocsInRAM, PeakDocCountWithoutFlush); } foreach (ThreadState threadState in notPending) { Assert.IsFalse(threadState.FlushPending); } }
public override void OnDelete(DocumentsWriterFlushControl control, ThreadState state) { List<ThreadState> pending = new List<ThreadState>(); List<ThreadState> notPending = new List<ThreadState>(); FindPending(control, pending, notPending); bool flushCurrent = state.FlushPending; ThreadState toFlush; if (state.FlushPending) { toFlush = state; } else if (FlushOnDeleteTerms() && state.DocumentsWriterPerThread.NumDeleteTerms() >= IWConfig.MaxBufferedDeleteTerms) { toFlush = state; } else { toFlush = null; } base.OnDelete(control, state); if (toFlush != null) { if (flushCurrent) { Assert.IsTrue(pending.Remove(toFlush)); } else { Assert.IsTrue(notPending.Remove(toFlush)); } Assert.IsTrue(toFlush.FlushPending); HasMarkedPending = true; } foreach (ThreadState threadState in notPending) { Assert.IsFalse(threadState.FlushPending); } }
protected internal virtual void AssertActiveBytesAfter(DocumentsWriterFlushControl flushControl) { IEnumerator<ThreadState> allActiveThreads = flushControl.AllActiveThreadStates(); long bytesUsed = 0; while (allActiveThreads.MoveNext()) { ThreadState next = allActiveThreads.Current; if (next.DocumentsWriterPerThread != null) { bytesUsed += next.DocumentsWriterPerThread.BytesUsed(); } } Assert.AreEqual(bytesUsed, flushControl.ActiveBytes()); }
internal DocumentsWriter(IndexWriter writer, LiveIndexWriterConfig config, Directory directory) { this.Directory = directory; this.LIWConfig = config; this.InfoStream = config.InfoStream; this.PerThreadPool = config.IndexerThreadPool; FlushPolicy = config.FlushPolicy; this.Writer = writer; this.Events = new ConcurrentQueue<Event>(); FlushControl = new DocumentsWriterFlushControl(this, config, writer.BufferedUpdatesStream); }
/// <summary> /// Marks the most ram consuming active <seealso cref="DocumentsWriterPerThread"/> flush /// pending /// </summary> protected internal virtual void MarkLargestWriterPending(DocumentsWriterFlushControl control, ThreadState perThreadState, long currentBytesPerThread) { control.FlushPending = FindLargestNonPendingWriter(control, perThreadState); }