/// <summary> /// Create and return a new <see cref="MergeThread"/> </summary> private MergeThread CreateTask(IndexWriter writer, MergePolicy.OneMerge merge) { var count = Interlocked.Increment(ref _mergeThreadCount); var name = string.Format("Lucene Merge Task #{0}", count); return(new MergeThread(name, writer, merge, writer.infoStream, Verbose, _manualResetEvent, HandleMergeException, DoMerge)); }
public virtual int Compare(MergeThread t1, MergeThread t2) { MergePolicy.OneMerge m1 = t1.CurrentMerge; MergePolicy.OneMerge m2 = t2.CurrentMerge; int c1 = m1 == null ? int.MaxValue : m1.TotalDocCount; int c2 = m2 == null ? int.MaxValue : m2.TotalDocCount; return(c2 - c1); }
/// <summary> /// Create and return a new <see cref="MergeThread"/> </summary> protected virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { lock (this) { MergeThread thread = new MergeThread(this, writer, merge); thread.SetThreadPriority((ThreadPriority)mergeThreadPriority); thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + m_mergeThreadCount++; return(thread); } }
/// <summary> /// Sole constructor. </summary> public MergeThread(string name, IndexWriter writer, MergePolicy.OneMerge startMerge, InfoStream logger, bool isLoggingEnabled, ManualResetEventSlim resetEvent, Action <Exception> exceptionHandler, Action <MergePolicy.OneMerge> doMerge) { Name = name; _cancellationTokenSource = new CancellationTokenSource(); _writer = writer; _startingMerge = startMerge; _logger = logger; _isLoggingEnabled = isLoggingEnabled; _resetEvent = resetEvent; _exceptionHandler = exceptionHandler; _doMerge = doMerge; }
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) // LUCENENET NOTE: This was internal in the original, but the base class is public so there isn't much choice here { lock (this) { while (true) { MergePolicy.OneMerge merge = writer.NextMerge(); if (merge == null) { break; } writer.Merge(merge); } } }
/// <summary> /// Create and return a new <see cref="MergeThread"/> </summary> protected virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { UninterruptableMonitor.Enter(this); try { MergeThread thread = new MergeThread(this, writer, merge); thread.SetThreadPriority((ThreadPriority)mergeThreadPriority); thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + m_mergeThreadCount++; return(thread); } finally { UninterruptableMonitor.Exit(this); } }
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) // LUCENENET NOTE: This was internal in the original, but the base class is public so there isn't much choice here { UninterruptableMonitor.Enter(this); try { while (true) { MergePolicy.OneMerge merge = writer.NextMerge(); if (merge is null) { break; } writer.Merge(merge); } } finally { UninterruptableMonitor.Exit(this); } }
/// <summary> /// Creates a <see cref="CheckAbort"/> instance. </summary> public CheckAbort(MergePolicy.OneMerge merge, Directory dir) { this.merge = merge; this.dir = dir; }
public override void Run() { // First time through the while loop we do the merge // that we were started with: MergePolicy.OneMerge merge = this.startMerge; try { if (outerInstance.IsVerbose) { outerInstance.Message(" merge thread: start"); } while (true) { RunningMerge = merge; outerInstance.DoMerge(merge); // Subsequent times through the loop we do any new // merge that writer says is necessary: merge = tWriter.NextMerge(); // Notify here in case any threads were stalled; // they will notice that the pending merge has // been pulled and possibly resume: UninterruptableMonitor.Enter(outerInstance); try { UninterruptableMonitor.PulseAll(outerInstance); } finally { UninterruptableMonitor.Exit(outerInstance); } if (merge != null) { outerInstance.UpdateMergeThreads(); if (outerInstance.IsVerbose) { outerInstance.Message(" merge thread: do another merge " + tWriter.SegString(merge.Segments)); } } else { break; } } if (outerInstance.IsVerbose) { outerInstance.Message(" merge thread: done"); } } catch (Exception exc) when(exc.IsThrowable()) { // Ignore the exception if it was due to abort: if (!(exc is MergePolicy.MergeAbortedException)) { //System.out.println(Thread.currentThread().getName() + ": CMS: exc"); //exc.printStackTrace(System.out); if (!outerInstance.suppressExceptions) { // suppressExceptions is normally only set during // testing. outerInstance.HandleMergeException(exc); } } } finally { done = true; UninterruptableMonitor.Enter(outerInstance); try { outerInstance.UpdateMergeThreads(); UninterruptableMonitor.PulseAll(outerInstance); } finally { UninterruptableMonitor.Exit(outerInstance); } } }
/// <summary> /// Sole constructor. </summary> public MergeThread(ConcurrentMergeScheduler outerInstance, IndexWriter writer, MergePolicy.OneMerge startMerge) { this.outerInstance = outerInstance; this.tWriter = writer; this.startMerge = startMerge; }
protected virtual void DoMerge(MergePolicy.OneMerge merge) { m_writer.Merge(merge); }
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) { UninterruptableMonitor.Enter(this); try { if (Debugging.AssertsEnabled) { Debugging.Assert(!UninterruptableMonitor.IsEntered(writer)); } this.m_writer = writer; InitMergeThreadPriority(); m_dir = writer.Directory; // First, quickly run through the newly proposed merges // and add any orthogonal merges (ie a merge not // involving segments already pending to be merged) to // the queue. If we are way behind on merging, many of // these newly proposed merges will likely already be // registered. if (IsVerbose) { Message("now merge"); Message(" index: " + writer.SegString()); } // Iterate, pulling from the IndexWriter's queue of // pending merges, until it's empty: while (true) { long startStallTime = 0; while (writer.HasPendingMerges() && MergeThreadCount >= maxMergeCount) { // this means merging has fallen too far behind: we // have already created maxMergeCount threads, and // now there's at least one more merge pending. // Note that only maxThreadCount of // those created merge threads will actually be // running; the rest will be paused (see // updateMergeThreads). We stall this producer // thread to prevent creation of new segments, // until merging has caught up: startStallTime = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results if (IsVerbose) { Message(" too many merges; stalling..."); } try { UninterruptableMonitor.Wait(this); } catch (Exception ie) when(ie.IsInterruptedException()) { throw new Util.ThreadInterruptedException(ie); } } if (IsVerbose) { if (startStallTime != 0) { Message(" stalled for " + ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - startStallTime) + " msec"); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results } } MergePolicy.OneMerge merge = writer.NextMerge(); if (merge == null) { if (IsVerbose) { Message(" no more merges pending; now return"); } return; } bool success = false; try { if (IsVerbose) { Message(" consider merge " + writer.SegString(merge.Segments)); } // OK to spawn a new merge thread to handle this // merge: MergeThread merger = GetMergeThread(writer, merge); m_mergeThreads.Add(merger); if (IsVerbose) { Message(" launch new thread [" + merger.Name + "]"); } merger.Start(); // Must call this after starting the thread else // the new thread is removed from mergeThreads // (since it's not alive yet): UpdateMergeThreads(); success = true; } finally { if (!success) { writer.MergeFinish(merge); } } } } finally { UninterruptableMonitor.Exit(this); } }
/// <summary> /// Called whenever the running merges have changed, to pause & unpause /// threads. This method sorts the merge threads by their merge size in /// descending order and then pauses/unpauses threads from first to last -- /// that way, smaller merges are guaranteed to run before larger ones. /// </summary> protected virtual void UpdateMergeThreads() { UninterruptableMonitor.Enter(this); try { // Only look at threads that are alive & not in the // process of stopping (ie have an active merge): IList <MergeThread> activeMerges = new JCG.List <MergeThread>(); int threadIdx = 0; while (threadIdx < m_mergeThreads.Count) { MergeThread mergeThread = m_mergeThreads[threadIdx]; if (!mergeThread.IsAlive) { // Prune any dead threads m_mergeThreads.RemoveAt(threadIdx); continue; } if (mergeThread.CurrentMerge != null) { activeMerges.Add(mergeThread); } threadIdx++; } // Sort the merge threads in descending order. CollectionUtil.TimSort(activeMerges, compareByMergeDocCount); int pri = mergeThreadPriority; int activeMergeCount = activeMerges.Count; for (threadIdx = 0; threadIdx < activeMergeCount; threadIdx++) { MergeThread mergeThread = activeMerges[threadIdx]; MergePolicy.OneMerge merge = mergeThread.CurrentMerge; if (merge == null) { continue; } // pause the thread if maxThreadCount is smaller than the number of merge threads. bool doPause = threadIdx < activeMergeCount - maxThreadCount; if (IsVerbose) { if (doPause != merge.IsPaused) { if (doPause) { Message("pause thread " + mergeThread.Name); } else { Message("unpause thread " + mergeThread.Name); } } } if (doPause != merge.IsPaused) { merge.SetPause(doPause); } if (!doPause) { if (IsVerbose) { Message("set priority of merge thread " + mergeThread.Name + " to " + pri); } mergeThread.SetThreadPriority((ThreadPriority)pri); pri = Math.Min((int)ThreadPriority.Highest, 1 + pri); } } } finally { UninterruptableMonitor.Exit(this); } }
private void Run(CancellationToken cancellationToken) { // First time through the while loop we do the merge // that we were started with: MergePolicy.OneMerge merge = _startingMerge; try { if (_isLoggingEnabled) { _logger.Message(COMPONENT_NAME, " merge thread: start"); } while (true && !cancellationToken.IsCancellationRequested) { RunningMerge = merge; // LUCENENET NOTE: We MUST call DoMerge(merge) instead of // _writer.Merge(merge) because the tests specifically look // for the method name DoMerge in the stack trace. _doMerge(merge); // Subsequent times through the loop we do any new // merge that writer says is necessary: merge = _writer.NextMerge(); // Notify here in case any threads were stalled; // they will notice that the pending merge has // been pulled and possibly resume: _resetEvent.Set(); if (merge != null) { if (_isLoggingEnabled) { _logger.Message(COMPONENT_NAME, " merge thread: do another merge " + _writer.SegString(merge.Segments)); } } else { break; } } if (_isLoggingEnabled) { _logger.Message(COMPONENT_NAME, " merge thread: done"); } } catch (Exception exc) { // Ignore the exception if it was due to abort: if (!(exc is MergePolicy.MergeAbortedException)) { //System.out.println(Thread.currentThread().getName() + ": CMS: exc"); //exc.printStackTrace(System.out) _exceptionHandler(exc); } } finally { _isDone = true; MergeThreadCompleted?.Invoke(this, EventArgs.Empty); } }
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) { using (_lock.Write()) { _writer = writer; _directory = writer.Directory; if (Verbose) { Message("now merge"); Message(" index: " + writer.SegString()); } // First, quickly run through the newly proposed merges // and add any orthogonal merges (ie a merge not // involving segments already pending to be merged) to // the queue. If we are way behind on merging, many of // these newly proposed merges will likely already be // registered. // Iterate, pulling from the IndexWriter's queue of // pending merges, until it's empty: while (true) { long startStallTime = 0; while (writer.HasPendingMerges() && MergeThreadCount >= MaxMergeCount) { // this means merging has fallen too far behind: we // have already created maxMergeCount threads, and // now there's at least one more merge pending. // Note that only maxThreadCount of // those created merge threads will actually be // running; the rest will be paused (see // updateMergeThreads). We stall this producer // thread to prevent creation of new segments, // until merging has caught up: startStallTime = Environment.TickCount; if (Verbose) { Message(" too many merges; stalling..."); } _manualResetEvent.Reset(); _manualResetEvent.Wait(); } if (Verbose) { if (startStallTime != 0) { Message(" stalled for " + (Environment.TickCount - startStallTime) + " msec"); } } MergePolicy.OneMerge merge = writer.NextMerge(); if (merge is null) { if (Verbose) { Message(" no more merges pending; now return"); } return; } bool success = false; try { if (Verbose) { Message(" consider merge " + writer.SegString(merge.Segments)); } // OK to spawn a new merge thread to handle this // merge: var merger = CreateTask(writer, merge); merger.MergeThreadCompleted += OnMergeThreadCompleted; _mergeThreads.Add(merger); if (Verbose) { Message(" launch new thread [" + merger.Name + "]"); } merger.Start(_taskScheduler); // Must call this after starting the thread else // the new thread is removed from mergeThreads // (since it's not alive yet): UpdateMergeThreads(); success = true; } finally { if (!success) { writer.MergeFinish(merge); } } } } }