public override void OnDelete(DocumentsWriterFlushControl control, ThreadState state) { if (FlushOnDeleteTerms) { // Flush this state by num del terms int maxBufferedDeleteTerms = m_indexWriterConfig.MaxBufferedDeleteTerms; if (control.NumGlobalTermDeletes >= maxBufferedDeleteTerms) { control.SetApplyAllDeletes(); } } if ((FlushOnRAM && control.DeleteBytesUsed > (1024 * 1024 * m_indexWriterConfig.RAMBufferSizeMB))) { control.SetApplyAllDeletes(); if (m_infoStream.IsEnabled("FP")) { m_infoStream.Message("FP", "force apply deletes bytesUsed=" + control.DeleteBytesUsed + " vs ramBuffer=" + (1024 * 1024 * m_indexWriterConfig.RAMBufferSizeMB)); } } }
public override void OnDelete(DocumentsWriterFlushControl control, ThreadState state) { if (FlushOnDeleteTerms()) { // Flush this state by num del terms int maxBufferedDeleteTerms = IWConfig.MaxBufferedDeleteTerms; if (control.NumGlobalTermDeletes >= maxBufferedDeleteTerms) { control.SetApplyAllDeletes(); } } if ((FlushOnRAM() && control.DeleteBytesUsed > (1024 * 1024 * IWConfig.RAMBufferSizeMB))) { control.SetApplyAllDeletes(); if (InfoStream.IsEnabled("FP")) { InfoStream.Message("FP", "force apply deletes bytesUsed=" + control.DeleteBytesUsed + " vs ramBuffer=" + (1024 * 1024 * IWConfig.RAMBufferSizeMB)); } } }
public override void OnInsert(DocumentsWriterFlushControl control, ThreadState state) { if (FlushOnDocCount && state.dwpt.NumDocsInRAM >= m_indexWriterConfig.MaxBufferedDocs) { // Flush this state by num docs control.SetFlushPending(state); } // flush by RAM else if (FlushOnRAM) { long limit = (long)(m_indexWriterConfig.RAMBufferSizeMB * 1024d * 1024d); long totalRam = control.ActiveBytes + control.DeleteBytesUsed; if (totalRam >= limit) { if (m_infoStream.IsEnabled("FP")) { m_infoStream.Message("FP", "flush: activeBytes=" + control.ActiveBytes + " deleteBytes=" + control.DeleteBytesUsed + " vs limit=" + limit); } MarkLargestWriterPending(control, state, totalRam); } } }
public override void OnInsert(DocumentsWriterFlushControl control, ThreadState state) { if (FlushOnDocCount() && state.Dwpt.NumDocsInRAM >= IWConfig.MaxBufferedDocs) { // Flush this state by num docs control.FlushPending = state; } // flush by RAM else if (FlushOnRAM()) { long limit = (long)(IWConfig.RAMBufferSizeMB * 1024d * 1024d); long totalRam = control.ActiveBytes() + control.DeleteBytesUsed; if (totalRam >= limit) { if (InfoStream.IsEnabled("FP")) { InfoStream.Message("FP", "flush: activeBytes=" + control.ActiveBytes() + " deleteBytes=" + control.DeleteBytesUsed + " vs limit=" + limit); } MarkLargestWriterPending(control, state, totalRam); } } }
/// <summary> /// Marks the most ram consuming active <see cref="DocumentsWriterPerThread"/> flush /// pending /// </summary> protected virtual void MarkLargestWriterPending(DocumentsWriterFlushControl control, ThreadState perThreadState, long currentBytesPerThread) { control.SetFlushPending(FindLargestNonPendingWriter(control, perThreadState)); }
/// <summary> /// Called for each document addition on the given <seealso cref="ThreadState"/>s /// <seealso cref="DocumentsWriterPerThread"/>. /// <p> /// Note: this method is synchronized by the given /// <seealso cref="DocumentsWriterFlushControl"/> and it is guaranteed that the calling /// thread holds the lock on the given <seealso cref="ThreadState"/> /// </summary> public abstract void OnInsert(DocumentsWriterFlushControl control, ThreadState state);
/// <summary> /// Returns the current most RAM consuming non-pending <seealso cref="ThreadState"/> with /// at least one indexed document. /// <p> /// this method will never return <code>null</code> /// </summary> protected internal virtual ThreadState FindLargestNonPendingWriter(DocumentsWriterFlushControl control, ThreadState perThreadState) { Debug.Assert(perThreadState.Dwpt.NumDocsInRAM > 0); long maxRamSoFar = perThreadState.BytesUsed; // the dwpt which needs to be flushed eventually ThreadState maxRamUsingThreadState = perThreadState; Debug.Assert(!perThreadState.FlushPending_Renamed, "DWPT should have flushed"); IEnumerator<ThreadState> activePerThreadsIterator = control.AllActiveThreadStates(); while (activePerThreadsIterator.MoveNext()) { ThreadState next = activePerThreadsIterator.Current; if (!next.FlushPending_Renamed) { long nextRam = next.BytesUsed; if (nextRam > maxRamSoFar && next.Dwpt.NumDocsInRAM > 0) { maxRamSoFar = nextRam; maxRamUsingThreadState = next; } } } Debug.Assert(AssertMessage("set largest ram consuming thread pending on lower watermark")); return maxRamUsingThreadState; }
/// <summary> /// Called for each document update on the given <seealso cref="ThreadState"/>'s /// <seealso cref="DocumentsWriterPerThread"/>. /// <p> /// Note: this method is called synchronized on the given /// <seealso cref="DocumentsWriterFlushControl"/> and it is guaranteed that the calling /// thread holds the lock on the given <seealso cref="ThreadState"/> /// </summary> public virtual void OnUpdate(DocumentsWriterFlushControl control, ThreadState state) { OnInsert(control, state); OnDelete(control, state); }
private void EnsureInitialized(ThreadState state) { if (state.Active && state.Dwpt == null) { FieldInfos.Builder infos = new FieldInfos.Builder(Writer.GlobalFieldNumberMap); state.Dwpt = new DocumentsWriterPerThread(Writer.NewSegmentName(), Directory, LIWConfig, InfoStream, DeleteQueue, infos); } }
private void AbortThreadState(ThreadState perThread, ISet<string> newFiles) { //Debug.Assert(perThread.HeldByCurrentThread); if (perThread.Active) // we might be closed { if (perThread.Initialized) { try { SubtractFlushedNumDocs(perThread.Dwpt.NumDocsInRAM); perThread.Dwpt.Abort(newFiles); } finally { perThread.Dwpt.CheckAndResetHasAborted(); FlushControl.DoOnAbort(perThread); } } else { FlushControl.DoOnAbort(perThread); } } else { Debug.Assert(Closed); } }
public override void OnInsert(DocumentsWriterFlushControl control, ThreadState state) { List<ThreadState> pending = new List<ThreadState>(); List<ThreadState> notPending = new List<ThreadState>(); FindPending(control, pending, notPending); bool flushCurrent = state.FlushPending; long activeBytes = control.ActiveBytes(); ThreadState toFlush; if (state.FlushPending) { toFlush = state; } else if (FlushOnDocCount() && state.DocumentsWriterPerThread.NumDocsInRAM >= IWConfig.MaxBufferedDocs) { toFlush = state; } else if (FlushOnRAM() && activeBytes >= (long)(IWConfig.RAMBufferSizeMB * 1024.0 * 1024.0)) { toFlush = FindLargestNonPendingWriter(control, state); Assert.IsFalse(toFlush.FlushPending); } else { toFlush = null; } base.OnInsert(control, state); if (toFlush != null) { if (flushCurrent) { Assert.IsTrue(pending.Remove(toFlush)); } else { Assert.IsTrue(notPending.Remove(toFlush)); } Assert.IsTrue(toFlush.FlushPending); HasMarkedPending = true; } else { PeakBytesWithoutFlush = Math.Max(activeBytes, PeakBytesWithoutFlush); PeakDocCountWithoutFlush = Math.Max(state.DocumentsWriterPerThread.NumDocsInRAM, PeakDocCountWithoutFlush); } foreach (ThreadState threadState in notPending) { Assert.IsFalse(threadState.FlushPending); } }
public override void OnDelete(DocumentsWriterFlushControl control, ThreadState state) { List<ThreadState> pending = new List<ThreadState>(); List<ThreadState> notPending = new List<ThreadState>(); FindPending(control, pending, notPending); bool flushCurrent = state.FlushPending; ThreadState toFlush; if (state.FlushPending) { toFlush = state; } else if (FlushOnDeleteTerms() && state.DocumentsWriterPerThread.NumDeleteTerms() >= IWConfig.MaxBufferedDeleteTerms) { toFlush = state; } else { toFlush = null; } base.OnDelete(control, state); if (toFlush != null) { if (flushCurrent) { Assert.IsTrue(pending.Remove(toFlush)); } else { Assert.IsTrue(notPending.Remove(toFlush)); } Assert.IsTrue(toFlush.FlushPending); HasMarkedPending = true; } foreach (ThreadState threadState in notPending) { Assert.IsFalse(threadState.FlushPending); } }
/// <summary> /// Marks the most ram consuming active <seealso cref="DocumentsWriterPerThread"/> flush /// pending /// </summary> protected internal virtual void MarkLargestWriterPending(DocumentsWriterFlushControl control, ThreadState perThreadState, long currentBytesPerThread) { control.FlushPending = FindLargestNonPendingWriter(control, perThreadState); }