public virtual void SetRunningMerge(MergePolicy.OneMerge merge)
{
lock (this)
{
runningMerge = merge;
}
}
/// <summary>
/// Create and return a new <see cref="MergeThread"/> </summary>
private MergeThread CreateTask(IndexWriter writer, MergePolicy.OneMerge merge)
{
var count = Interlocked.Increment(ref _mergeThreadCount);
var name = string.Format("Lucene Merge Task #{0}", count);
return(new MergeThread(name, writer, merge, writer.infoStream, Verbose, _manualResetEvent, HandleMergeException));
}
override public void Run()
{
// First time through the while loop we do the merge
// that we were started with:
MergePolicy.OneMerge merge = this.startMerge;
try
{
Enclosing_Instance.Message(" merge thread: start");
while (true)
{
SetRunningMerge(merge);
Enclosing_Instance.DoMerge(merge);
// Subsequent times through the loop we do any new
// merge that writer says is necessary:
merge = writer.GetNextMerge();
if (merge != null)
{
writer.MergeInit(merge);
Enclosing_Instance.Message(" merge thread: do another merge " + merge.SegString(Enclosing_Instance.dir));
}
else
{
break;
}
}
Enclosing_Instance.Message(" merge thread: done");
}
catch (System.Exception exc)
{
// Ignore the exception if it was due to abort:
if (!(exc is MergePolicy.MergeAbortedException))
{
lock (Enclosing_Instance)
{
Enclosing_Instance.exceptions.Add(exc);
}
if (!Enclosing_Instance.suppressExceptions)
{
// suppressExceptions is normally only set during
// testing.
Lucene.Net.Index.ConcurrentMergeScheduler.anyExceptions = true;
Enclosing_Instance.HandleMergeException(exc);
}
}
}
finally
{
lock (Enclosing_Instance)
{
System.Threading.Monitor.PulseAll(Enclosing_Instance);
bool removed = Enclosing_Instance.mergeThreads.Remove(this);
System.Diagnostics.Debug.Assert(removed);
}
}
}
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound)
{
UninterruptableMonitor.Enter(this);
try
{
while (true)
{
MergePolicy.OneMerge merge = writer.NextMerge();
if (merge == null)
{
break;
}
for (int i = 0; i < merge.Segments.Count; i++)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(merge.Segments[i].Info.DocCount < 20);
}
}
writer.Merge(merge);
}
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge, IState state)
{
InitBlock(enclosingInstance);
this.writer = writer;
this.startMerge = startMerge;
this.state = state;
}
public override System.String ToString()
{
MergePolicy.OneMerge merge = GetRunningMerge();
if (merge == null)
{
merge = startMerge;
}
return("merge thread: " + merge.SegString(Enclosing_Instance.dir));
}
/// <summary>
/// Create and return a new MergeThread.
/// </summary>
/// <param name="writer"></param>
/// <param name="merge"></param>
/// <returns></returns>
virtual protected MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge)
{
MergeThread thread = new MergeThread(this, writer, merge);
thread.SetThreadPriority(mergeThreadPriority);
thread.IsBackground = true;
thread.Name = "Lucene Merge Thread #" + mergeThreadCount++;
return(thread);
}
public virtual int Compare(MergeThread t1, MergeThread t2)
{
MergePolicy.OneMerge m1 = t1.CurrentMerge;
MergePolicy.OneMerge m2 = t2.CurrentMerge;
int c1 = m1 == null ? int.MaxValue : m1.TotalDocCount;
int c2 = m2 == null ? int.MaxValue : m2.TotalDocCount;
return(c2 - c1);
}
internal SegmentMerger(IndexWriter writer, System.String name, MergePolicy.OneMerge merge)
{
InitBlock();
directory = writer.GetDirectory();
segment = name;
if (merge != null)
{
checkAbort = new CheckAbort(merge, directory);
}
termIndexInterval = writer.GetTermIndexInterval();
}
/// <summary>
/// Create and return a new MergeThread </summary>
protected internal virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge)
{
lock (this)
{
MergeThread thread = new MergeThread(this, writer, merge);
thread.ThreadPriority = MergeThreadPriority_Renamed;
thread.IsBackground = true;
thread.Name = "Lucene Merge Thread #" + MergeThreadCount_Renamed++;
return(thread);
}
}
/// <summary>Create and return a new MergeThread </summary>
protected internal virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge, IState state)
{
lock (this)
{
var thread = new MergeThread(this, writer, merge, state);
thread.SetThreadPriority(mergeThreadPriority);
thread.IsBackground = true;
thread.Name = "Lucene Merge Thread #" + mergeThreadCount++;
return(thread);
}
}
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound)
{
MergePolicy.OneMerge merge = null;
while ((merge = writer.NextMerge()) != null)
{
if (VERBOSE)
{
Console.WriteLine("executing merge " + merge.SegString(writer.Directory));
}
writer.Merge(merge);
}
}
public override void Merge(MergePolicy.OneMerge merge)
{
if (merge.MaxNumSegments != -1 && (first || merge.Segments.Count == 1))
{
first = false;
if (Verbose)
{
Console.WriteLine("TEST: maxNumSegments merge");
}
mergeCount.IncrementAndGet();
}
base.Merge(merge);
}
/// <summary>
/// Sole constructor. </summary>
public MergeThread(string name, IndexWriter writer, MergePolicy.OneMerge startMerge,
InfoStream logger, bool isLoggingEnabled,
ManualResetEventSlim resetEvent, Action <Exception> exceptionHandler)
{
Name = name;
_cancellationTokenSource = new CancellationTokenSource();
_writer = writer;
_startingMerge = startMerge;
_logger = logger;
_isLoggingEnabled = isLoggingEnabled;
_resetEvent = resetEvent;
_exceptionHandler = exceptionHandler;
}
internal SegmentMerger(IndexWriter writer, System.String name, MergePolicy.OneMerge merge)
{
InitBlock();
directory = writer.Directory;
segment = name;
if (merge != null)
{
checkAbort = new CheckAbort(merge, directory);
}
else
{
checkAbort = new AnonymousClassCheckAbort1(this, null, null);
}
termIndexInterval = writer.TermIndexInterval;
}
/// <summary>Just do the merges in sequence. We do this
/// "synchronized" so that even if the application is using
/// multiple threads, only one merge may run at a time.
/// </summary>
public override void Merge(IndexWriter writer)
{
lock (this)
{
while (true)
{
MergePolicy.OneMerge merge = writer.GetNextMerge();
if (merge == null)
{
break;
}
writer.Merge(merge);
}
}
}
/// <summary>
/// Just do the merges in sequence. We do this
/// "synchronized" so that even if the application is using
/// multiple threads, only one merge may run at a time.
/// </summary>
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) // LUCENENET NOTE: This was internal in the original, but the base class is public so there isn't much choice here
{
lock (this)
{
while (true)
{
MergePolicy.OneMerge merge = writer.NextMerge();
if (merge == null)
{
break;
}
writer.Merge(merge);
}
}
}
/// <summary>
/// Create and return a new <see cref="MergeThread"/> </summary>
protected virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge)
{
UninterruptableMonitor.Enter(this);
try
{
MergeThread thread = new MergeThread(this, writer, merge);
thread.SetThreadPriority((ThreadPriority)mergeThreadPriority);
thread.IsBackground = true;
thread.Name = "Lucene Merge Thread #" + m_mergeThreadCount++;
return(thread);
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound)
{
lock (this)
{
while (true)
{
MergePolicy.OneMerge merge = writer.NextMerge();
if (merge == null)
{
break;
}
for (int i = 0; i < merge.Segments.Count; i++)
{
Debug.Assert(merge.Segments[i].Info.DocCount < 20);
}
writer.Merge(merge);
}
}
}
protected override void DoMerge(MergePolicy.OneMerge merge)
{
try
{
// Stall all incoming merges until we see
// maxMergeCount:
int count = runningMergeCount.IncrementAndGet();
try
{
Assert.IsTrue(count <= maxMergeCount, "count=" + count + " vs maxMergeCount=" + maxMergeCount);
enoughMergesWaiting.Signal();
// Stall this merge until we see exactly
// maxMergeCount merges waiting
while (true)
{
// wait for 10 milliseconds
if (enoughMergesWaiting.Wait(new TimeSpan(0, 0, 0, 0, 10)) || failed)
{
break;
}
}
// Then sleep a bit to give a chance for the bug
// (too many pending merges) to appear:
Thread.Sleep(20);
base.DoMerge(merge);
}
finally
{
runningMergeCount.DecrementAndGet();
}
}
catch (Exception /*t*/)
{
failed.Value = (true);
m_writer.MergeFinish(merge);
// LUCENENET specific - throwing an exception on a background thread causes the test
// runner to crash on .NET Core 2.0.
//throw new Exception(t.ToString(), t);
}
}
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) // LUCENENET NOTE: This was internal in the original, but the base class is public so there isn't much choice here
{
UninterruptableMonitor.Enter(this);
try
{
while (true)
{
MergePolicy.OneMerge merge = writer.NextMerge();
if (merge is null)
{
break;
}
writer.Merge(merge);
}
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
private int MergeThreadCount(bool excludeDone)
{
lock (this)
{
int count = 0;
int numThreads = mergeThreads.Count;
for (int i = 0; i < numThreads; i++)
{
MergeThread t = (MergeThread)mergeThreads[i];
if (t.IsAlive)
{
MergePolicy.OneMerge runningMerge = t.GetRunningMerge();
if (!excludeDone || (runningMerge != null && !runningMerge.mergeDone))
{
count++;
}
}
}
return(count);
}
}
protected override void DoMerge(MergePolicy.OneMerge merge)
{
try
{
// Stall all incoming merges until we see
// maxMergeCount:
int count = runningMergeCount.IncrementAndGet();
try
{
Assert.IsTrue(count <= maxMergeCount, "count=" + count + " vs maxMergeCount=" + maxMergeCount);
enoughMergesWaiting.Signal();
// Stall this merge until we see exactly
// maxMergeCount merges waiting
while (true)
{
// wait for 10 milliseconds
if (enoughMergesWaiting.Wait(new TimeSpan(0, 0, 0, 0, 10)) || failed)
{
break;
}
}
// Then sleep a bit to give a chance for the bug
// (too many pending merges) to appear:
Thread.Sleep(20);
base.DoMerge(merge);
}
finally
{
runningMergeCount.DecrementAndGet();
}
}
catch (Exception t)
{
failed.Value = (true);
m_writer.MergeFinish(merge);
// LUCENENET NOTE: ThreadJob takes care of propagating the exception to the calling thread
throw new Exception(t.ToString(), t);
}
}
protected internal override void DoMerge(MergePolicy.OneMerge merge)
{
try
{
// Stall all incoming merges until we see
// maxMergeCount:
int count = RunningMergeCount.IncrementAndGet();
try
{
Assert.IsTrue(count <= MaxMergeCount, "count=" + count + " vs maxMergeCount=" + MaxMergeCount);
EnoughMergesWaiting.Signal();
// Stall this merge until we see exactly
// maxMergeCount merges waiting
while (true)
{
// wait for 10 milliseconds
if (EnoughMergesWaiting.Wait(new TimeSpan(0, 0, 0, 0, 10)) || Failed.Get())
{
break;
}
}
// Then sleep a bit to give a chance for the bug
// (too many pending merges) to appear:
Thread.Sleep(20);
base.DoMerge(merge);
}
finally
{
RunningMergeCount.DecrementAndGet();
}
}
catch (Exception t)
{
Failed.Set(true);
Writer.MergeFinish(merge);
throw new Exception(t.Message, t);
}
}
public override void Run()
{
// First time through the while loop we do the merge
// that we were started with:
MergePolicy.OneMerge merge = this.startMerge;
try
{
if (outerInstance.IsVerbose)
{
outerInstance.Message(" merge thread: start");
}
while (true)
{
RunningMerge = merge;
outerInstance.DoMerge(merge);
// Subsequent times through the loop we do any new
// merge that writer says is necessary:
merge = tWriter.NextMerge();
// Notify here in case any threads were stalled;
// they will notice that the pending merge has
// been pulled and possibly resume:
lock (outerInstance)
{
Monitor.PulseAll(outerInstance);
}
if (merge != null)
{
outerInstance.UpdateMergeThreads();
if (outerInstance.IsVerbose)
{
outerInstance.Message(" merge thread: do another merge " + tWriter.SegString(merge.Segments));
}
}
else
{
break;
}
}
if (outerInstance.IsVerbose)
{
outerInstance.Message(" merge thread: done");
}
}
catch (Exception exc)
{
// Ignore the exception if it was due to abort:
if (!(exc is MergePolicy.MergeAbortedException))
{
//System.out.println(Thread.currentThread().getName() + ": CMS: exc");
//exc.printStackTrace(System.out);
if (!outerInstance.suppressExceptions)
{
// suppressExceptions is normally only set during
// testing.
outerInstance.HandleMergeException(exc);
}
}
}
finally
{
done = true;
lock (outerInstance)
{
outerInstance.UpdateMergeThreads();
Monitor.PulseAll(outerInstance);
}
}
}
/// <summary>
/// Sole constructor. </summary>
public MergeThread(ConcurrentMergeScheduler outerInstance, IndexWriter writer, MergePolicy.OneMerge startMerge)
{
this.outerInstance = outerInstance;
this.tWriter = writer;
this.startMerge = startMerge;
}
protected virtual void DoMerge(MergePolicy.OneMerge merge)
{
m_writer.Merge(merge);
}
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound)
{
lock (this)
{
//Debug.Assert(!Thread.holdsLock(writer));
this.m_writer = writer;
InitMergeThreadPriority();
m_dir = writer.Directory;
// First, quickly run through the newly proposed merges
// and add any orthogonal merges (ie a merge not
// involving segments already pending to be merged) to
// the queue. If we are way behind on merging, many of
// these newly proposed merges will likely already be
// registered.
if (IsVerbose)
{
Message("now merge");
Message(" index: " + writer.SegString());
}
// Iterate, pulling from the IndexWriter's queue of
// pending merges, until it's empty:
while (true)
{
long startStallTime = 0;
while (writer.HasPendingMerges() && MergeThreadCount >= maxMergeCount)
{
// this means merging has fallen too far behind: we
// have already created maxMergeCount threads, and
// now there's at least one more merge pending.
// Note that only maxThreadCount of
// those created merge threads will actually be
// running; the rest will be paused (see
// updateMergeThreads). We stall this producer
// thread to prevent creation of new segments,
// until merging has caught up:
startStallTime = Environment.TickCount;
if (IsVerbose)
{
Message(" too many merges; stalling...");
}
//try
//{
Monitor.Wait(this);
//}
//catch (ThreadInterruptedException ie) // LUCENENET NOTE: Senseless to catch and rethrow the same exception type
//{
// throw new ThreadInterruptedException(ie.ToString(), ie);
//}
}
if (IsVerbose)
{
if (startStallTime != 0)
{
Message(" stalled for " + (Environment.TickCount - startStallTime) + " msec");
}
}
MergePolicy.OneMerge merge = writer.NextMerge();
if (merge == null)
{
if (IsVerbose)
{
Message(" no more merges pending; now return");
}
return;
}
bool success = false;
try
{
if (IsVerbose)
{
Message(" consider merge " + writer.SegString(merge.Segments));
}
// OK to spawn a new merge thread to handle this
// merge:
MergeThread merger = GetMergeThread(writer, merge);
m_mergeThreads.Add(merger);
if (IsVerbose)
{
Message(" launch new thread [" + merger.Name + "]");
}
merger.Start();
// Must call this after starting the thread else
// the new thread is removed from mergeThreads
// (since it's not alive yet):
UpdateMergeThreads();
success = true;
}
finally
{
if (!success)
{
writer.MergeFinish(merge);
}
}
}
}
}
/// <summary>
/// Called whenever the running merges have changed, to pause & unpause
/// threads. This method sorts the merge threads by their merge size in
/// descending order and then pauses/unpauses threads from first to last --
/// that way, smaller merges are guaranteed to run before larger ones.
/// </summary>
protected virtual void UpdateMergeThreads()
{
lock (this)
{
// Only look at threads that are alive & not in the
// process of stopping (ie have an active merge):
IList <MergeThread> activeMerges = new List <MergeThread>();
int threadIdx = 0;
while (threadIdx < m_mergeThreads.Count)
{
MergeThread mergeThread = m_mergeThreads[threadIdx];
if (!mergeThread.IsAlive)
{
// Prune any dead threads
m_mergeThreads.RemoveAt(threadIdx);
continue;
}
if (mergeThread.CurrentMerge != null)
{
activeMerges.Add(mergeThread);
}
threadIdx++;
}
// Sort the merge threads in descending order.
CollectionUtil.TimSort(activeMerges, compareByMergeDocCount);
int pri = mergeThreadPriority;
int activeMergeCount = activeMerges.Count;
for (threadIdx = 0; threadIdx < activeMergeCount; threadIdx++)
{
MergeThread mergeThread = activeMerges[threadIdx];
MergePolicy.OneMerge merge = mergeThread.CurrentMerge;
if (merge == null)
{
continue;
}
// pause the thread if maxThreadCount is smaller than the number of merge threads.
bool doPause = threadIdx < activeMergeCount - maxThreadCount;
if (IsVerbose)
{
if (doPause != merge.IsPaused)
{
if (doPause)
{
Message("pause thread " + mergeThread.Name);
}
else
{
Message("unpause thread " + mergeThread.Name);
}
}
}
if (doPause != merge.IsPaused)
{
merge.SetPause(doPause);
}
if (!doPause)
{
if (IsVerbose)
{
Message("set priority of merge thread " + mergeThread.Name + " to " + pri);
}
mergeThread.SetThreadPriority((ThreadPriority)pri);
pri = Math.Min((int)ThreadPriority.Highest, 1 + pri);
}
}
}
}
public virtual void SetRunningMerge(MergePolicy.OneMerge merge)
{
lock (this)
{
runningMerge = merge;
}
}
/// <summary>
/// Sole constructor. </summary>
public MergeThread(string name, IndexWriter writer, MergePolicy.OneMerge startMerge,
InfoStream logger, bool isLoggingEnabled,
ManualResetEventSlim resetEvent, Action<Exception> exceptionHandler)
{
Name = name;
_cancellationTokenSource = new CancellationTokenSource();
_writer = writer;
_startingMerge = startMerge;
_logger = logger;
_isLoggingEnabled = isLoggingEnabled;
_resetEvent = resetEvent;
_exceptionHandler = exceptionHandler;
}
/* If any of our segments are using a directory != ours
* then copy them over. Currently this is only used by
* addIndexesNoOptimize(). */
private void CopyExternalSegments()
{
bool any = false;
while (true)
{
SegmentInfo info = null;
MergePolicy.OneMerge merge = null;
lock (this)
{
int numSegments = segmentInfos.Count;
for (int i = 0; i < numSegments; i++)
{
info = segmentInfos.Info(i);
if (info.dir != directory)
{
merge = new MergePolicy.OneMerge(segmentInfos.Range(i, 1 + i), info.GetUseCompoundFile());
break;
}
}
}
if (merge != null)
{
if (RegisterMerge(merge))
{
pendingMerges.Remove(merge);
runningMerges.Add(merge, merge);
any = true;
Merge(merge);
}
else
{
// This means there is a bug in the
// MergeScheduler. MergeSchedulers in general are
// not allowed to run a merge involving segments
// external to this IndexWriter's directory in the
// background because this would put the index
// into an inconsistent state (where segmentInfos
// has been written with such external segments
// that an IndexReader would fail to load).
throw new MergePolicy.MergeException("segment \"" + info.name + " exists in external directory yet the MergeScheduler executed the merge in a separate thread");
}
}
else
{
// No more external segments
break;
}
}
if (any)
// Sometimes, on copying an external segment over,
// more merges may become necessary:
mergeScheduler.Merge(this);
}
public CheckAbort(MergePolicy.OneMerge merge, Directory dir)
{
this.merge = merge;
this.dir = dir;
}
public Action<double> Work; //.NET
public CheckAbort(MergePolicy.OneMerge merge, Directory dir)
{
this.merge = merge;
this.dir = dir;
this.Work = Internal_Work;
}
//.NET
public CheckAbort(MergePolicy.OneMerge merge, Directory dir, Action<double> work)
{
this.merge = merge;
this.dir = dir;
this.Work = work;
}
/* If any of our segments are using a directory != ours
* then we have to either copy them over one by one, merge
* them (if merge policy has chosen to) or wait until
* currently running merges (in the background) complete.
* We don't return until the SegmentInfos has no more
* external segments. Currently this is only used by
* addIndexesNoOptimize(). */
private void ResolveExternalSegments()
{
bool any = false;
bool done = false;
while (!done)
{
SegmentInfo info = null;
MergePolicy.OneMerge merge = null;
lock (this)
{
if (stopMerges)
throw new MergePolicy.MergeAbortedException("rollback() was called or addIndexes* hit an unhandled exception");
int numSegments = segmentInfos.Count;
done = true;
for (int i = 0; i < numSegments; i++)
{
info = segmentInfos.Info(i);
if (info.dir != directory)
{
done = false;
MergePolicy.OneMerge newMerge = new MergePolicy.OneMerge(segmentInfos.Range(i, 1 + i), mergePolicy is LogMergePolicy && GetUseCompoundFile());
// Returns true if no running merge conflicts
// with this one (and, records this merge as
// pending), ie, this segment is not currently
// being merged:
if (RegisterMerge(newMerge))
{
merge = newMerge;
// If this segment is not currently being
// merged, then advance it to running & run
// the merge ourself (below):
pendingMerges.Remove(merge); // {{Aroush-2.9}} From Mike Garski: this is an O(n) op... is that an issue?
runningMerges.Add(merge);
break;
}
}
}
if (!done && merge == null)
// We are not yet done (external segments still
// exist in segmentInfos), yet, all such segments
// are currently "covered" by a pending or running
// merge. We now try to grab any pending merge
// that involves external segments:
merge = GetNextExternalMerge();
if (!done && merge == null)
// We are not yet done, and, all external segments
// fall under merges that the merge scheduler is
// currently running. So, we now wait and check
// back to see if the merge has completed.
DoWait();
}
if (merge != null)
{
any = true;
Merge(merge);
}
}
if (any)
// Sometimes, on copying an external segment over,
// more merges may become necessary:
mergeScheduler.Merge(this);
}
protected internal override void DoMerge(MergePolicy.OneMerge merge)
{
TotMergedBytes += merge.TotalBytesSize();
base.DoMerge(merge);
}
public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge)
{
InitBlock(enclosingInstance);
this.writer = writer;
this.startMerge = startMerge;
}
