//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: //ORIGINAL LINE: public void merge(IndexWriter writer, MergeTrigger trigger, boolean newMergesFound) throws java.io.IOException public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) { while (true) { MergePolicy.OneMerge merge = writer.NextMerge; if (merge == null) { return; } bool success = false; try { MergeThread mergeThread = getMergeThread(writer, merge); _writerTaskCounter.increment(); PooledConcurrentMergePool.MergeThreadsPool.submit(MergeTask(mergeThread)); success = true; } finally { if (!success) { writer.mergeFinish(merge); _writerTaskCounter.decrement(); } } } }
/// <summary> /// Create and return a new MergeThread. /// </summary> /// <param name="writer"></param> /// <param name="merge"></param> /// <returns></returns> virtual protected MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { MergeThread thread = new MergeThread(this, writer, merge); thread.SetThreadPriority(mergeThreadPriority); thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + mergeThreadCount++; return(thread); }
/// <summary> /// Create and return a new MergeThread </summary> protected internal virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { lock (this) { MergeThread thread = new MergeThread(this, writer, merge); thread.ThreadPriority = MergeThreadPriority_Renamed; thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + MergeThreadCount_Renamed++; return(thread); } }
/// <summary>Create and return a new MergeThread </summary> protected internal virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge, IState state) { lock (this) { var thread = new MergeThread(this, writer, merge, state); thread.SetThreadPriority(mergeThreadPriority); thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + mergeThreadCount++; return(thread); } }
/// <summary> /// Wait for any running merge threads to finish. This call is not interruptible as used by <see cref="Dispose(bool)"/>. </summary> public virtual void Sync() { bool interrupted = false; try { while (true) { MergeThread toSync = null; UninterruptableMonitor.Enter(this); try { foreach (MergeThread t in m_mergeThreads) { if (t != null && t.IsAlive) { toSync = t; break; } } } finally { UninterruptableMonitor.Exit(this); } if (toSync != null) { try { toSync.Join(); } catch (Exception ie) when(ie.IsInterruptedException()) { // ignore this Exception, we will retry until all threads are dead interrupted = true; } } else { break; } } } finally { // finally, restore interrupt status: if (interrupted) { Thread.CurrentThread.Interrupt(); } } }
/// <summary> /// Wait for any running merge threads to finish. This call is not interruptible as used by <see cref="Dispose(bool)"/>. </summary> public virtual void Sync() { bool interrupted = false; try { while (true) { MergeThread toSync = null; lock (this) { foreach (MergeThread t in m_mergeThreads) { if (t != null && t.IsAlive) { toSync = t; break; } } } if (toSync != null) { try { toSync.Join(); } #pragma warning disable 168 catch (ThreadInterruptedException ie) #pragma warning restore 168 { // ignore this Exception, we will retry until all threads are dead interrupted = true; } } else { break; } } } finally { // finally, restore interrupt status: if (interrupted) { Thread.CurrentThread.Interrupt(); } } }
/// <summary> /// Create and return a new <see cref="MergeThread"/> </summary> protected virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { UninterruptableMonitor.Enter(this); try { MergeThread thread = new MergeThread(this, writer, merge); thread.SetThreadPriority((ThreadPriority)mergeThreadPriority); thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + m_mergeThreadCount++; return(thread); } finally { UninterruptableMonitor.Exit(this); } }
/// <summary>Return the priority that merge threads run at. </summary> public virtual void SetMergeThreadPriority(int pri) { lock (this) { if (pri > (int)System.Threading.ThreadPriority.Highest || pri < (int)System.Threading.ThreadPriority.Lowest) { throw new System.ArgumentException("priority must be in range " + (int)System.Threading.ThreadPriority.Lowest + " .. " + (int)System.Threading.ThreadPriority.Highest + " inclusive"); } mergeThreadPriority = pri; int numThreads = MergeThreadCount(); for (int i = 0; i < numThreads; i++) { MergeThread merge = (MergeThread)mergeThreads[i]; merge.SetThreadPriority(pri); } } }
private int MergeThreadCount(bool excludeDone) { lock (this) { int count = 0; int numThreads = mergeThreads.Count; for (int i = 0; i < numThreads; i++) { MergeThread t = (MergeThread)mergeThreads[i]; if (t.IsAlive) { MergePolicy.OneMerge runningMerge = t.GetRunningMerge(); if (!excludeDone || (runningMerge != null && !runningMerge.mergeDone)) { count++; } } } return(count); } }
/// <summary> /// Called whenever the running merges have changed, to pause & unpause /// threads. This method sorts the merge threads by their merge size in /// descending order and then pauses/unpauses threads from first to last -- /// that way, smaller merges are guaranteed to run before larger ones. /// </summary> protected virtual void UpdateMergeThreads() { lock (this) { // Only look at threads that are alive & not in the // process of stopping (ie have an active merge): IList <MergeThread> activeMerges = new List <MergeThread>(); int threadIdx = 0; while (threadIdx < m_mergeThreads.Count) { MergeThread mergeThread = m_mergeThreads[threadIdx]; if (!mergeThread.IsAlive) { // Prune any dead threads m_mergeThreads.RemoveAt(threadIdx); continue; } if (mergeThread.CurrentMerge != null) { activeMerges.Add(mergeThread); } threadIdx++; } // Sort the merge threads in descending order. CollectionUtil.TimSort(activeMerges, compareByMergeDocCount); int pri = mergeThreadPriority; int activeMergeCount = activeMerges.Count; for (threadIdx = 0; threadIdx < activeMergeCount; threadIdx++) { MergeThread mergeThread = activeMerges[threadIdx]; MergePolicy.OneMerge merge = mergeThread.CurrentMerge; if (merge == null) { continue; } // pause the thread if maxThreadCount is smaller than the number of merge threads. bool doPause = threadIdx < activeMergeCount - maxThreadCount; if (IsVerbose) { if (doPause != merge.IsPaused) { if (doPause) { Message("pause thread " + mergeThread.Name); } else { Message("unpause thread " + mergeThread.Name); } } } if (doPause != merge.IsPaused) { merge.SetPause(doPause); } if (!doPause) { if (IsVerbose) { Message("set priority of merge thread " + mergeThread.Name + " to " + pri); } mergeThread.SetThreadPriority((ThreadPriority)pri); pri = Math.Min((int)ThreadPriority.Highest, 1 + pri); } } } }
public override void Merge(IndexWriter writer) { this.writer = writer; InitMergeThreadPriority(); dir = writer.GetDirectory(); // First, quickly run through the newly proposed merges // and add any orthogonal merges (ie a merge not // involving segments already pending to be merged) to // the queue. If we are way behind on merging, many of // these newly proposed merges will likely already be // registered. Message("now merge"); Message(" index: " + writer.SegString()); // Iterate, pulling from the IndexWriter's queue of // pending merges, until its empty: while (true) { // TODO: we could be careful about which merges to do in // the BG (eg maybe the "biggest" ones) vs FG, which // merges to do first (the easiest ones?), etc. MergePolicy.OneMerge merge = writer.GetNextMerge(); if (merge == null) { Message(" no more merges pending; now return"); return; } // We do this w/ the primary thread to keep // deterministic assignment of segment names writer.MergeInit(merge); Message(" consider merge " + merge.SegString(dir)); if (merge.isExternal) { Message(" merge involves segments from an external directory; now run in foreground"); } else { lock (this) { if (MergeThreadCount() < maxThreadCount) { // OK to spawn a new merge thread to handle this // merge: MergeThread merger = new MergeThread(this, writer, merge); mergeThreads.Add(merger); Message(" launch new thread [" + merger.Name + "]"); merger.SetThreadPriority(mergeThreadPriority); merger.IsBackground = true; merger.Start(); continue; } else { Message(" too many merge threads running; run merge in foreground"); } } } // Too many merge threads already running, so we do // this in the foreground of the calling thread writer.Merge(merge); } }
public override void Merge(IndexWriter writer) { this.writer = writer; InitMergeThreadPriority(); dir = writer.GetDirectory(); // First, quickly run through the newly proposed merges // and add any orthogonal merges (ie a merge not // involving segments already pending to be merged) to // the queue. If we are way behind on merging, many of // these newly proposed merges will likely already be // registered. Message("now merge"); Message(" index: " + writer.SegString()); // Iterate, pulling from the IndexWriter's queue of // pending merges, until its empty: while (true) { // TODO: we could be careful about which merges to do in // the BG (eg maybe the "biggest" ones) vs FG, which // merges to do first (the easiest ones?), etc. MergePolicy.OneMerge merge = writer.GetNextMerge(); if (merge == null) { Message(" no more merges pending; now return"); return ; } // We do this w/ the primary thread to keep // deterministic assignment of segment names writer.MergeInit(merge); Message(" consider merge " + merge.SegString(dir)); if (merge.isExternal) { Message(" merge involves segments from an external directory; now run in foreground"); } else { lock (this) { if (MergeThreadCount() < maxThreadCount) { // OK to spawn a new merge thread to handle this // merge: MergeThread merger = new MergeThread(this, writer, merge); mergeThreads.Add(merger); Message(" launch new thread [" + merger.Name + "]"); merger.SetThreadPriority(mergeThreadPriority); merger.IsBackground = true; merger.Start(); continue; } else Message(" too many merge threads running; run merge in foreground"); } } // Too many merge threads already running, so we do // this in the foreground of the calling thread writer.Merge(merge); } }
public override void Merge(IndexWriter writer) { // TODO: .NET doesn't support this // assert !Thread.holdsLock(writer); this.writer = writer; InitMergeThreadPriority(); dir = writer.Directory; // First, quickly run through the newly proposed merges // and add any orthogonal merges (ie a merge not // involving segments already pending to be merged) to // the queue. If we are way behind on merging, many of // these newly proposed merges will likely already be // registered. if (Verbose()) { Message("now merge"); Message(" index: " + writer.SegString()); } // Iterate, pulling from the IndexWriter's queue of // pending merges, until it's empty: while (true) { // TODO: we could be careful about which merges to do in // the BG (eg maybe the "biggest" ones) vs FG, which // merges to do first (the easiest ones?), etc. MergePolicy.OneMerge merge = writer.GetNextMerge(); if (merge == null) { if (Verbose()) { Message(" no more merges pending; now return"); } return; } // We do this w/ the primary thread to keep // deterministic assignment of segment names writer.MergeInit(merge); bool success = false; try { lock (this) { while (MergeThreadCount() >= _maxThreadCount) { if (Verbose()) { Message(" too many merge threads running; stalling..."); } System.Threading.Monitor.Wait(this); } if (Verbose()) { Message(" consider merge " + merge.SegString(dir)); } System.Diagnostics.Debug.Assert(MergeThreadCount() < _maxThreadCount); // OK to spawn a new merge thread to handle this // merge: MergeThread merger = GetMergeThread(writer, merge); mergeThreads.Add(merger); if (Verbose()) { Message(" launch new thread [" + merger.Name + "]"); } merger.Start(); success = true; } } finally { if (!success) { writer.MergeFinish(merge); } } } }
internal MergeTask(MergeThread mergeThread, LongAdder taskCounter) { this.MergeThread = mergeThread; this.TaskCounter = taskCounter; }
private ThreadStart MergeTask(MergeThread mergeThread) { return(new MergeTask(mergeThread, _writerTaskCounter)); }
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) { lock (this) { //Debug.Assert(!Thread.holdsLock(writer)); this.m_writer = writer; InitMergeThreadPriority(); m_dir = writer.Directory; // First, quickly run through the newly proposed merges // and add any orthogonal merges (ie a merge not // involving segments already pending to be merged) to // the queue. If we are way behind on merging, many of // these newly proposed merges will likely already be // registered. if (IsVerbose) { Message("now merge"); Message(" index: " + writer.SegString()); } // Iterate, pulling from the IndexWriter's queue of // pending merges, until it's empty: while (true) { long startStallTime = 0; while (writer.HasPendingMerges() && MergeThreadCount >= maxMergeCount) { // this means merging has fallen too far behind: we // have already created maxMergeCount threads, and // now there's at least one more merge pending. // Note that only maxThreadCount of // those created merge threads will actually be // running; the rest will be paused (see // updateMergeThreads). We stall this producer // thread to prevent creation of new segments, // until merging has caught up: startStallTime = Environment.TickCount; if (IsVerbose) { Message(" too many merges; stalling..."); } //try //{ Monitor.Wait(this); //} //catch (ThreadInterruptedException ie) // LUCENENET NOTE: Senseless to catch and rethrow the same exception type //{ // throw new ThreadInterruptedException(ie.ToString(), ie); //} } if (IsVerbose) { if (startStallTime != 0) { Message(" stalled for " + (Environment.TickCount - startStallTime) + " msec"); } } MergePolicy.OneMerge merge = writer.NextMerge(); if (merge == null) { if (IsVerbose) { Message(" no more merges pending; now return"); } return; } bool success = false; try { if (IsVerbose) { Message(" consider merge " + writer.SegString(merge.Segments)); } // OK to spawn a new merge thread to handle this // merge: MergeThread merger = GetMergeThread(writer, merge); m_mergeThreads.Add(merger); if (IsVerbose) { Message(" launch new thread [" + merger.Name + "]"); } merger.Start(); // Must call this after starting the thread else // the new thread is removed from mergeThreads // (since it's not alive yet): UpdateMergeThreads(); success = true; } finally { if (!success) { writer.MergeFinish(merge); } } } } }
/// <summary>Create and return a new MergeThread </summary> protected internal virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { lock (this) { MergeThread thread = new MergeThread(this, writer, merge); thread.SetThreadPriority(mergeThreadPriority); thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + mergeThreadCount++; return thread; } }