public virtual void SetRunningMerge(MergePolicy.OneMerge merge) { lock (this) { runningMerge = merge; } }
/// <summary> /// Create and return a new <see cref="MergeThread"/> </summary> private MergeThread CreateTask(IndexWriter writer, MergePolicy.OneMerge merge) { var count = Interlocked.Increment(ref _mergeThreadCount); var name = string.Format("Lucene Merge Task #{0}", count); return(new MergeThread(name, writer, merge, writer.infoStream, Verbose, _manualResetEvent, HandleMergeException)); }
override public void Run() { // First time through the while loop we do the merge // that we were started with: MergePolicy.OneMerge merge = this.startMerge; try { Enclosing_Instance.Message(" merge thread: start"); while (true) { SetRunningMerge(merge); Enclosing_Instance.DoMerge(merge); // Subsequent times through the loop we do any new // merge that writer says is necessary: merge = writer.GetNextMerge(); if (merge != null) { writer.MergeInit(merge); Enclosing_Instance.Message(" merge thread: do another merge " + merge.SegString(Enclosing_Instance.dir)); } else { break; } } Enclosing_Instance.Message(" merge thread: done"); } catch (System.Exception exc) { // Ignore the exception if it was due to abort: if (!(exc is MergePolicy.MergeAbortedException)) { lock (Enclosing_Instance) { Enclosing_Instance.exceptions.Add(exc); } if (!Enclosing_Instance.suppressExceptions) { // suppressExceptions is normally only set during // testing. Lucene.Net.Index.ConcurrentMergeScheduler.anyExceptions = true; Enclosing_Instance.HandleMergeException(exc); } } } finally { lock (Enclosing_Instance) { System.Threading.Monitor.PulseAll(Enclosing_Instance); bool removed = Enclosing_Instance.mergeThreads.Remove(this); System.Diagnostics.Debug.Assert(removed); } } }
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) { UninterruptableMonitor.Enter(this); try { while (true) { MergePolicy.OneMerge merge = writer.NextMerge(); if (merge == null) { break; } for (int i = 0; i < merge.Segments.Count; i++) { if (Debugging.AssertsEnabled) { Debugging.Assert(merge.Segments[i].Info.DocCount < 20); } } writer.Merge(merge); } } finally { UninterruptableMonitor.Exit(this); } }
public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge, IState state) { InitBlock(enclosingInstance); this.writer = writer; this.startMerge = startMerge; this.state = state; }
public override System.String ToString() { MergePolicy.OneMerge merge = GetRunningMerge(); if (merge == null) { merge = startMerge; } return("merge thread: " + merge.SegString(Enclosing_Instance.dir)); }
/// <summary> /// Create and return a new MergeThread. /// </summary> /// <param name="writer"></param> /// <param name="merge"></param> /// <returns></returns> virtual protected MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { MergeThread thread = new MergeThread(this, writer, merge); thread.SetThreadPriority(mergeThreadPriority); thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + mergeThreadCount++; return(thread); }
public virtual int Compare(MergeThread t1, MergeThread t2) { MergePolicy.OneMerge m1 = t1.CurrentMerge; MergePolicy.OneMerge m2 = t2.CurrentMerge; int c1 = m1 == null ? int.MaxValue : m1.TotalDocCount; int c2 = m2 == null ? int.MaxValue : m2.TotalDocCount; return(c2 - c1); }
internal SegmentMerger(IndexWriter writer, System.String name, MergePolicy.OneMerge merge) { InitBlock(); directory = writer.GetDirectory(); segment = name; if (merge != null) { checkAbort = new CheckAbort(merge, directory); } termIndexInterval = writer.GetTermIndexInterval(); }
/// <summary> /// Create and return a new MergeThread </summary> protected internal virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { lock (this) { MergeThread thread = new MergeThread(this, writer, merge); thread.ThreadPriority = MergeThreadPriority_Renamed; thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + MergeThreadCount_Renamed++; return(thread); } }
/// <summary>Create and return a new MergeThread </summary> protected internal virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge, IState state) { lock (this) { var thread = new MergeThread(this, writer, merge, state); thread.SetThreadPriority(mergeThreadPriority); thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + mergeThreadCount++; return(thread); } }
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) { MergePolicy.OneMerge merge = null; while ((merge = writer.NextMerge()) != null) { if (VERBOSE) { Console.WriteLine("executing merge " + merge.SegString(writer.Directory)); } writer.Merge(merge); } }
public override void Merge(MergePolicy.OneMerge merge) { if (merge.MaxNumSegments != -1 && (first || merge.Segments.Count == 1)) { first = false; if (Verbose) { Console.WriteLine("TEST: maxNumSegments merge"); } mergeCount.IncrementAndGet(); } base.Merge(merge); }
/// <summary> /// Sole constructor. </summary> public MergeThread(string name, IndexWriter writer, MergePolicy.OneMerge startMerge, InfoStream logger, bool isLoggingEnabled, ManualResetEventSlim resetEvent, Action <Exception> exceptionHandler) { Name = name; _cancellationTokenSource = new CancellationTokenSource(); _writer = writer; _startingMerge = startMerge; _logger = logger; _isLoggingEnabled = isLoggingEnabled; _resetEvent = resetEvent; _exceptionHandler = exceptionHandler; }
internal SegmentMerger(IndexWriter writer, System.String name, MergePolicy.OneMerge merge) { InitBlock(); directory = writer.Directory; segment = name; if (merge != null) { checkAbort = new CheckAbort(merge, directory); } else { checkAbort = new AnonymousClassCheckAbort1(this, null, null); } termIndexInterval = writer.TermIndexInterval; }
/// <summary>Just do the merges in sequence. We do this /// "synchronized" so that even if the application is using /// multiple threads, only one merge may run at a time. /// </summary> public override void Merge(IndexWriter writer) { lock (this) { while (true) { MergePolicy.OneMerge merge = writer.GetNextMerge(); if (merge == null) { break; } writer.Merge(merge); } } }
/// <summary> /// Just do the merges in sequence. We do this /// "synchronized" so that even if the application is using /// multiple threads, only one merge may run at a time. /// </summary> public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) // LUCENENET NOTE: This was internal in the original, but the base class is public so there isn't much choice here { lock (this) { while (true) { MergePolicy.OneMerge merge = writer.NextMerge(); if (merge == null) { break; } writer.Merge(merge); } } }
/// <summary> /// Create and return a new <see cref="MergeThread"/> </summary> protected virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) { UninterruptableMonitor.Enter(this); try { MergeThread thread = new MergeThread(this, writer, merge); thread.SetThreadPriority((ThreadPriority)mergeThreadPriority); thread.IsBackground = true; thread.Name = "Lucene Merge Thread #" + m_mergeThreadCount++; return(thread); } finally { UninterruptableMonitor.Exit(this); } }
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) { lock (this) { while (true) { MergePolicy.OneMerge merge = writer.NextMerge(); if (merge == null) { break; } for (int i = 0; i < merge.Segments.Count; i++) { Debug.Assert(merge.Segments[i].Info.DocCount < 20); } writer.Merge(merge); } } }
protected override void DoMerge(MergePolicy.OneMerge merge) { try { // Stall all incoming merges until we see // maxMergeCount: int count = runningMergeCount.IncrementAndGet(); try { Assert.IsTrue(count <= maxMergeCount, "count=" + count + " vs maxMergeCount=" + maxMergeCount); enoughMergesWaiting.Signal(); // Stall this merge until we see exactly // maxMergeCount merges waiting while (true) { // wait for 10 milliseconds if (enoughMergesWaiting.Wait(new TimeSpan(0, 0, 0, 0, 10)) || failed) { break; } } // Then sleep a bit to give a chance for the bug // (too many pending merges) to appear: Thread.Sleep(20); base.DoMerge(merge); } finally { runningMergeCount.DecrementAndGet(); } } catch (Exception /*t*/) { failed.Value = (true); m_writer.MergeFinish(merge); // LUCENENET specific - throwing an exception on a background thread causes the test // runner to crash on .NET Core 2.0. //throw new Exception(t.ToString(), t); } }
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) // LUCENENET NOTE: This was internal in the original, but the base class is public so there isn't much choice here { UninterruptableMonitor.Enter(this); try { while (true) { MergePolicy.OneMerge merge = writer.NextMerge(); if (merge is null) { break; } writer.Merge(merge); } } finally { UninterruptableMonitor.Exit(this); } }
private int MergeThreadCount(bool excludeDone) { lock (this) { int count = 0; int numThreads = mergeThreads.Count; for (int i = 0; i < numThreads; i++) { MergeThread t = (MergeThread)mergeThreads[i]; if (t.IsAlive) { MergePolicy.OneMerge runningMerge = t.GetRunningMerge(); if (!excludeDone || (runningMerge != null && !runningMerge.mergeDone)) { count++; } } } return(count); } }
protected override void DoMerge(MergePolicy.OneMerge merge) { try { // Stall all incoming merges until we see // maxMergeCount: int count = runningMergeCount.IncrementAndGet(); try { Assert.IsTrue(count <= maxMergeCount, "count=" + count + " vs maxMergeCount=" + maxMergeCount); enoughMergesWaiting.Signal(); // Stall this merge until we see exactly // maxMergeCount merges waiting while (true) { // wait for 10 milliseconds if (enoughMergesWaiting.Wait(new TimeSpan(0, 0, 0, 0, 10)) || failed) { break; } } // Then sleep a bit to give a chance for the bug // (too many pending merges) to appear: Thread.Sleep(20); base.DoMerge(merge); } finally { runningMergeCount.DecrementAndGet(); } } catch (Exception t) { failed.Value = (true); m_writer.MergeFinish(merge); // LUCENENET NOTE: ThreadJob takes care of propagating the exception to the calling thread throw new Exception(t.ToString(), t); } }
protected internal override void DoMerge(MergePolicy.OneMerge merge) { try { // Stall all incoming merges until we see // maxMergeCount: int count = RunningMergeCount.IncrementAndGet(); try { Assert.IsTrue(count <= MaxMergeCount, "count=" + count + " vs maxMergeCount=" + MaxMergeCount); EnoughMergesWaiting.Signal(); // Stall this merge until we see exactly // maxMergeCount merges waiting while (true) { // wait for 10 milliseconds if (EnoughMergesWaiting.Wait(new TimeSpan(0, 0, 0, 0, 10)) || Failed.Get()) { break; } } // Then sleep a bit to give a chance for the bug // (too many pending merges) to appear: Thread.Sleep(20); base.DoMerge(merge); } finally { RunningMergeCount.DecrementAndGet(); } } catch (Exception t) { Failed.Set(true); Writer.MergeFinish(merge); throw new Exception(t.Message, t); } }
public override void Run() { // First time through the while loop we do the merge // that we were started with: MergePolicy.OneMerge merge = this.startMerge; try { if (outerInstance.IsVerbose) { outerInstance.Message(" merge thread: start"); } while (true) { RunningMerge = merge; outerInstance.DoMerge(merge); // Subsequent times through the loop we do any new // merge that writer says is necessary: merge = tWriter.NextMerge(); // Notify here in case any threads were stalled; // they will notice that the pending merge has // been pulled and possibly resume: lock (outerInstance) { Monitor.PulseAll(outerInstance); } if (merge != null) { outerInstance.UpdateMergeThreads(); if (outerInstance.IsVerbose) { outerInstance.Message(" merge thread: do another merge " + tWriter.SegString(merge.Segments)); } } else { break; } } if (outerInstance.IsVerbose) { outerInstance.Message(" merge thread: done"); } } catch (Exception exc) { // Ignore the exception if it was due to abort: if (!(exc is MergePolicy.MergeAbortedException)) { //System.out.println(Thread.currentThread().getName() + ": CMS: exc"); //exc.printStackTrace(System.out); if (!outerInstance.suppressExceptions) { // suppressExceptions is normally only set during // testing. outerInstance.HandleMergeException(exc); } } } finally { done = true; lock (outerInstance) { outerInstance.UpdateMergeThreads(); Monitor.PulseAll(outerInstance); } } }
/// <summary> /// Sole constructor. </summary> public MergeThread(ConcurrentMergeScheduler outerInstance, IndexWriter writer, MergePolicy.OneMerge startMerge) { this.outerInstance = outerInstance; this.tWriter = writer; this.startMerge = startMerge; }
protected virtual void DoMerge(MergePolicy.OneMerge merge) { m_writer.Merge(merge); }
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) { lock (this) { //Debug.Assert(!Thread.holdsLock(writer)); this.m_writer = writer; InitMergeThreadPriority(); m_dir = writer.Directory; // First, quickly run through the newly proposed merges // and add any orthogonal merges (ie a merge not // involving segments already pending to be merged) to // the queue. If we are way behind on merging, many of // these newly proposed merges will likely already be // registered. if (IsVerbose) { Message("now merge"); Message(" index: " + writer.SegString()); } // Iterate, pulling from the IndexWriter's queue of // pending merges, until it's empty: while (true) { long startStallTime = 0; while (writer.HasPendingMerges() && MergeThreadCount >= maxMergeCount) { // this means merging has fallen too far behind: we // have already created maxMergeCount threads, and // now there's at least one more merge pending. // Note that only maxThreadCount of // those created merge threads will actually be // running; the rest will be paused (see // updateMergeThreads). We stall this producer // thread to prevent creation of new segments, // until merging has caught up: startStallTime = Environment.TickCount; if (IsVerbose) { Message(" too many merges; stalling..."); } //try //{ Monitor.Wait(this); //} //catch (ThreadInterruptedException ie) // LUCENENET NOTE: Senseless to catch and rethrow the same exception type //{ // throw new ThreadInterruptedException(ie.ToString(), ie); //} } if (IsVerbose) { if (startStallTime != 0) { Message(" stalled for " + (Environment.TickCount - startStallTime) + " msec"); } } MergePolicy.OneMerge merge = writer.NextMerge(); if (merge == null) { if (IsVerbose) { Message(" no more merges pending; now return"); } return; } bool success = false; try { if (IsVerbose) { Message(" consider merge " + writer.SegString(merge.Segments)); } // OK to spawn a new merge thread to handle this // merge: MergeThread merger = GetMergeThread(writer, merge); m_mergeThreads.Add(merger); if (IsVerbose) { Message(" launch new thread [" + merger.Name + "]"); } merger.Start(); // Must call this after starting the thread else // the new thread is removed from mergeThreads // (since it's not alive yet): UpdateMergeThreads(); success = true; } finally { if (!success) { writer.MergeFinish(merge); } } } } }
/// <summary> /// Called whenever the running merges have changed, to pause & unpause /// threads. This method sorts the merge threads by their merge size in /// descending order and then pauses/unpauses threads from first to last -- /// that way, smaller merges are guaranteed to run before larger ones. /// </summary> protected virtual void UpdateMergeThreads() { lock (this) { // Only look at threads that are alive & not in the // process of stopping (ie have an active merge): IList <MergeThread> activeMerges = new List <MergeThread>(); int threadIdx = 0; while (threadIdx < m_mergeThreads.Count) { MergeThread mergeThread = m_mergeThreads[threadIdx]; if (!mergeThread.IsAlive) { // Prune any dead threads m_mergeThreads.RemoveAt(threadIdx); continue; } if (mergeThread.CurrentMerge != null) { activeMerges.Add(mergeThread); } threadIdx++; } // Sort the merge threads in descending order. CollectionUtil.TimSort(activeMerges, compareByMergeDocCount); int pri = mergeThreadPriority; int activeMergeCount = activeMerges.Count; for (threadIdx = 0; threadIdx < activeMergeCount; threadIdx++) { MergeThread mergeThread = activeMerges[threadIdx]; MergePolicy.OneMerge merge = mergeThread.CurrentMerge; if (merge == null) { continue; } // pause the thread if maxThreadCount is smaller than the number of merge threads. bool doPause = threadIdx < activeMergeCount - maxThreadCount; if (IsVerbose) { if (doPause != merge.IsPaused) { if (doPause) { Message("pause thread " + mergeThread.Name); } else { Message("unpause thread " + mergeThread.Name); } } } if (doPause != merge.IsPaused) { merge.SetPause(doPause); } if (!doPause) { if (IsVerbose) { Message("set priority of merge thread " + mergeThread.Name + " to " + pri); } mergeThread.SetThreadPriority((ThreadPriority)pri); pri = Math.Min((int)ThreadPriority.Highest, 1 + pri); } } } }
public virtual void SetRunningMerge(MergePolicy.OneMerge merge) { lock (this) { runningMerge = merge; } }
/// <summary> /// Sole constructor. </summary> public MergeThread(string name, IndexWriter writer, MergePolicy.OneMerge startMerge, InfoStream logger, bool isLoggingEnabled, ManualResetEventSlim resetEvent, Action<Exception> exceptionHandler) { Name = name; _cancellationTokenSource = new CancellationTokenSource(); _writer = writer; _startingMerge = startMerge; _logger = logger; _isLoggingEnabled = isLoggingEnabled; _resetEvent = resetEvent; _exceptionHandler = exceptionHandler; }
/* If any of our segments are using a directory != ours * then copy them over. Currently this is only used by * addIndexesNoOptimize(). */ private void CopyExternalSegments() { bool any = false; while (true) { SegmentInfo info = null; MergePolicy.OneMerge merge = null; lock (this) { int numSegments = segmentInfos.Count; for (int i = 0; i < numSegments; i++) { info = segmentInfos.Info(i); if (info.dir != directory) { merge = new MergePolicy.OneMerge(segmentInfos.Range(i, 1 + i), info.GetUseCompoundFile()); break; } } } if (merge != null) { if (RegisterMerge(merge)) { pendingMerges.Remove(merge); runningMerges.Add(merge, merge); any = true; Merge(merge); } else { // This means there is a bug in the // MergeScheduler. MergeSchedulers in general are // not allowed to run a merge involving segments // external to this IndexWriter's directory in the // background because this would put the index // into an inconsistent state (where segmentInfos // has been written with such external segments // that an IndexReader would fail to load). throw new MergePolicy.MergeException("segment \"" + info.name + " exists in external directory yet the MergeScheduler executed the merge in a separate thread"); } } else { // No more external segments break; } } if (any) // Sometimes, on copying an external segment over, // more merges may become necessary: mergeScheduler.Merge(this); }
public CheckAbort(MergePolicy.OneMerge merge, Directory dir) { this.merge = merge; this.dir = dir; }
public Action<double> Work; //.NET public CheckAbort(MergePolicy.OneMerge merge, Directory dir) { this.merge = merge; this.dir = dir; this.Work = Internal_Work; }
//.NET public CheckAbort(MergePolicy.OneMerge merge, Directory dir, Action<double> work) { this.merge = merge; this.dir = dir; this.Work = work; }
/* If any of our segments are using a directory != ours * then we have to either copy them over one by one, merge * them (if merge policy has chosen to) or wait until * currently running merges (in the background) complete. * We don't return until the SegmentInfos has no more * external segments. Currently this is only used by * addIndexesNoOptimize(). */ private void ResolveExternalSegments() { bool any = false; bool done = false; while (!done) { SegmentInfo info = null; MergePolicy.OneMerge merge = null; lock (this) { if (stopMerges) throw new MergePolicy.MergeAbortedException("rollback() was called or addIndexes* hit an unhandled exception"); int numSegments = segmentInfos.Count; done = true; for (int i = 0; i < numSegments; i++) { info = segmentInfos.Info(i); if (info.dir != directory) { done = false; MergePolicy.OneMerge newMerge = new MergePolicy.OneMerge(segmentInfos.Range(i, 1 + i), mergePolicy is LogMergePolicy && GetUseCompoundFile()); // Returns true if no running merge conflicts // with this one (and, records this merge as // pending), ie, this segment is not currently // being merged: if (RegisterMerge(newMerge)) { merge = newMerge; // If this segment is not currently being // merged, then advance it to running & run // the merge ourself (below): pendingMerges.Remove(merge); // {{Aroush-2.9}} From Mike Garski: this is an O(n) op... is that an issue? runningMerges.Add(merge); break; } } } if (!done && merge == null) // We are not yet done (external segments still // exist in segmentInfos), yet, all such segments // are currently "covered" by a pending or running // merge. We now try to grab any pending merge // that involves external segments: merge = GetNextExternalMerge(); if (!done && merge == null) // We are not yet done, and, all external segments // fall under merges that the merge scheduler is // currently running. So, we now wait and check // back to see if the merge has completed. DoWait(); } if (merge != null) { any = true; Merge(merge); } } if (any) // Sometimes, on copying an external segment over, // more merges may become necessary: mergeScheduler.Merge(this); }
protected internal override void DoMerge(MergePolicy.OneMerge merge) { TotMergedBytes += merge.TotalBytesSize(); base.DoMerge(merge); }
public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge) { InitBlock(enclosingInstance); this.writer = writer; this.startMerge = startMerge; }