/// <summary>
/// Create and return a new <see cref="MergeThread"/> </summary>
private MergeThread CreateTask(IndexWriter writer, MergePolicy.OneMerge merge)
{
var count = Interlocked.Increment(ref _mergeThreadCount);
var name = string.Format("Lucene Merge Task #{0}", count);
return(new MergeThread(name, writer, merge, writer.infoStream, Verbose, _manualResetEvent, HandleMergeException, DoMerge));
}
public virtual int Compare(MergeThread t1, MergeThread t2)
{
MergePolicy.OneMerge m1 = t1.CurrentMerge;
MergePolicy.OneMerge m2 = t2.CurrentMerge;
int c1 = m1 == null ? int.MaxValue : m1.TotalDocCount;
int c2 = m2 == null ? int.MaxValue : m2.TotalDocCount;
return(c2 - c1);
}
/// <summary>
/// Create and return a new <see cref="MergeThread"/> </summary>
protected virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge)
{
lock (this)
{
MergeThread thread = new MergeThread(this, writer, merge);
thread.SetThreadPriority((ThreadPriority)mergeThreadPriority);
thread.IsBackground = true;
thread.Name = "Lucene Merge Thread #" + m_mergeThreadCount++;
return(thread);
}
}
/// <summary>
/// Sole constructor. </summary>
public MergeThread(string name, IndexWriter writer, MergePolicy.OneMerge startMerge,
InfoStream logger, bool isLoggingEnabled,
ManualResetEventSlim resetEvent, Action <Exception> exceptionHandler, Action <MergePolicy.OneMerge> doMerge)
{
Name = name;
_cancellationTokenSource = new CancellationTokenSource();
_writer = writer;
_startingMerge = startMerge;
_logger = logger;
_isLoggingEnabled = isLoggingEnabled;
_resetEvent = resetEvent;
_exceptionHandler = exceptionHandler;
_doMerge = doMerge;
}
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) // LUCENENET NOTE: This was internal in the original, but the base class is public so there isn't much choice here
{
lock (this)
{
while (true)
{
MergePolicy.OneMerge merge = writer.NextMerge();
if (merge == null)
{
break;
}
writer.Merge(merge);
}
}
}
/// <summary>
/// Create and return a new <see cref="MergeThread"/> </summary>
protected virtual MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge)
{
UninterruptableMonitor.Enter(this);
try
{
MergeThread thread = new MergeThread(this, writer, merge);
thread.SetThreadPriority((ThreadPriority)mergeThreadPriority);
thread.IsBackground = true;
thread.Name = "Lucene Merge Thread #" + m_mergeThreadCount++;
return(thread);
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound) // LUCENENET NOTE: This was internal in the original, but the base class is public so there isn't much choice here
{
UninterruptableMonitor.Enter(this);
try
{
while (true)
{
MergePolicy.OneMerge merge = writer.NextMerge();
if (merge is null)
{
break;
}
writer.Merge(merge);
}
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
/// <summary>
/// Creates a <see cref="CheckAbort"/> instance. </summary>
public CheckAbort(MergePolicy.OneMerge merge, Directory dir)
{
this.merge = merge;
this.dir = dir;
}
public override void Run()
{
// First time through the while loop we do the merge
// that we were started with:
MergePolicy.OneMerge merge = this.startMerge;
try
{
if (outerInstance.IsVerbose)
{
outerInstance.Message(" merge thread: start");
}
while (true)
{
RunningMerge = merge;
outerInstance.DoMerge(merge);
// Subsequent times through the loop we do any new
// merge that writer says is necessary:
merge = tWriter.NextMerge();
// Notify here in case any threads were stalled;
// they will notice that the pending merge has
// been pulled and possibly resume:
UninterruptableMonitor.Enter(outerInstance);
try
{
UninterruptableMonitor.PulseAll(outerInstance);
}
finally
{
UninterruptableMonitor.Exit(outerInstance);
}
if (merge != null)
{
outerInstance.UpdateMergeThreads();
if (outerInstance.IsVerbose)
{
outerInstance.Message(" merge thread: do another merge " + tWriter.SegString(merge.Segments));
}
}
else
{
break;
}
}
if (outerInstance.IsVerbose)
{
outerInstance.Message(" merge thread: done");
}
}
catch (Exception exc) when(exc.IsThrowable())
{
// Ignore the exception if it was due to abort:
if (!(exc is MergePolicy.MergeAbortedException))
{
//System.out.println(Thread.currentThread().getName() + ": CMS: exc");
//exc.printStackTrace(System.out);
if (!outerInstance.suppressExceptions)
{
// suppressExceptions is normally only set during
// testing.
outerInstance.HandleMergeException(exc);
}
}
}
finally
{
done = true;
UninterruptableMonitor.Enter(outerInstance);
try
{
outerInstance.UpdateMergeThreads();
UninterruptableMonitor.PulseAll(outerInstance);
}
finally
{
UninterruptableMonitor.Exit(outerInstance);
}
}
}
/// <summary>
/// Sole constructor. </summary>
public MergeThread(ConcurrentMergeScheduler outerInstance, IndexWriter writer, MergePolicy.OneMerge startMerge)
{
this.outerInstance = outerInstance;
this.tWriter = writer;
this.startMerge = startMerge;
}
protected virtual void DoMerge(MergePolicy.OneMerge merge)
{
m_writer.Merge(merge);
}
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound)
{
UninterruptableMonitor.Enter(this);
try
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!UninterruptableMonitor.IsEntered(writer));
}
this.m_writer = writer;
InitMergeThreadPriority();
m_dir = writer.Directory;
// First, quickly run through the newly proposed merges
// and add any orthogonal merges (ie a merge not
// involving segments already pending to be merged) to
// the queue. If we are way behind on merging, many of
// these newly proposed merges will likely already be
// registered.
if (IsVerbose)
{
Message("now merge");
Message(" index: " + writer.SegString());
}
// Iterate, pulling from the IndexWriter's queue of
// pending merges, until it's empty:
while (true)
{
long startStallTime = 0;
while (writer.HasPendingMerges() && MergeThreadCount >= maxMergeCount)
{
// this means merging has fallen too far behind: we
// have already created maxMergeCount threads, and
// now there's at least one more merge pending.
// Note that only maxThreadCount of
// those created merge threads will actually be
// running; the rest will be paused (see
// updateMergeThreads). We stall this producer
// thread to prevent creation of new segments,
// until merging has caught up:
startStallTime = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
if (IsVerbose)
{
Message(" too many merges; stalling...");
}
try
{
UninterruptableMonitor.Wait(this);
}
catch (Exception ie) when(ie.IsInterruptedException())
{
throw new Util.ThreadInterruptedException(ie);
}
}
if (IsVerbose)
{
if (startStallTime != 0)
{
Message(" stalled for " + ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - startStallTime) + " msec"); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
}
}
MergePolicy.OneMerge merge = writer.NextMerge();
if (merge == null)
{
if (IsVerbose)
{
Message(" no more merges pending; now return");
}
return;
}
bool success = false;
try
{
if (IsVerbose)
{
Message(" consider merge " + writer.SegString(merge.Segments));
}
// OK to spawn a new merge thread to handle this
// merge:
MergeThread merger = GetMergeThread(writer, merge);
m_mergeThreads.Add(merger);
if (IsVerbose)
{
Message(" launch new thread [" + merger.Name + "]");
}
merger.Start();
// Must call this after starting the thread else
// the new thread is removed from mergeThreads
// (since it's not alive yet):
UpdateMergeThreads();
success = true;
}
finally
{
if (!success)
{
writer.MergeFinish(merge);
}
}
}
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
/// <summary>
/// Called whenever the running merges have changed, to pause & unpause
/// threads. This method sorts the merge threads by their merge size in
/// descending order and then pauses/unpauses threads from first to last --
/// that way, smaller merges are guaranteed to run before larger ones.
/// </summary>
protected virtual void UpdateMergeThreads()
{
UninterruptableMonitor.Enter(this);
try
{
// Only look at threads that are alive & not in the
// process of stopping (ie have an active merge):
IList <MergeThread> activeMerges = new JCG.List <MergeThread>();
int threadIdx = 0;
while (threadIdx < m_mergeThreads.Count)
{
MergeThread mergeThread = m_mergeThreads[threadIdx];
if (!mergeThread.IsAlive)
{
// Prune any dead threads
m_mergeThreads.RemoveAt(threadIdx);
continue;
}
if (mergeThread.CurrentMerge != null)
{
activeMerges.Add(mergeThread);
}
threadIdx++;
}
// Sort the merge threads in descending order.
CollectionUtil.TimSort(activeMerges, compareByMergeDocCount);
int pri = mergeThreadPriority;
int activeMergeCount = activeMerges.Count;
for (threadIdx = 0; threadIdx < activeMergeCount; threadIdx++)
{
MergeThread mergeThread = activeMerges[threadIdx];
MergePolicy.OneMerge merge = mergeThread.CurrentMerge;
if (merge == null)
{
continue;
}
// pause the thread if maxThreadCount is smaller than the number of merge threads.
bool doPause = threadIdx < activeMergeCount - maxThreadCount;
if (IsVerbose)
{
if (doPause != merge.IsPaused)
{
if (doPause)
{
Message("pause thread " + mergeThread.Name);
}
else
{
Message("unpause thread " + mergeThread.Name);
}
}
}
if (doPause != merge.IsPaused)
{
merge.SetPause(doPause);
}
if (!doPause)
{
if (IsVerbose)
{
Message("set priority of merge thread " + mergeThread.Name + " to " + pri);
}
mergeThread.SetThreadPriority((ThreadPriority)pri);
pri = Math.Min((int)ThreadPriority.Highest, 1 + pri);
}
}
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
private void Run(CancellationToken cancellationToken)
{
// First time through the while loop we do the merge
// that we were started with:
MergePolicy.OneMerge merge = _startingMerge;
try
{
if (_isLoggingEnabled)
{
_logger.Message(COMPONENT_NAME, " merge thread: start");
}
while (true && !cancellationToken.IsCancellationRequested)
{
RunningMerge = merge;
// LUCENENET NOTE: We MUST call DoMerge(merge) instead of
// _writer.Merge(merge) because the tests specifically look
// for the method name DoMerge in the stack trace.
_doMerge(merge);
// Subsequent times through the loop we do any new
// merge that writer says is necessary:
merge = _writer.NextMerge();
// Notify here in case any threads were stalled;
// they will notice that the pending merge has
// been pulled and possibly resume:
_resetEvent.Set();
if (merge != null)
{
if (_isLoggingEnabled)
{
_logger.Message(COMPONENT_NAME, " merge thread: do another merge " + _writer.SegString(merge.Segments));
}
}
else
{
break;
}
}
if (_isLoggingEnabled)
{
_logger.Message(COMPONENT_NAME, " merge thread: done");
}
}
catch (Exception exc)
{
// Ignore the exception if it was due to abort:
if (!(exc is MergePolicy.MergeAbortedException))
{
//System.out.println(Thread.currentThread().getName() + ": CMS: exc");
//exc.printStackTrace(System.out)
_exceptionHandler(exc);
}
}
finally
{
_isDone = true;
MergeThreadCompleted?.Invoke(this, EventArgs.Empty);
}
}
public override void Merge(IndexWriter writer, MergeTrigger trigger, bool newMergesFound)
{
using (_lock.Write())
{
_writer = writer;
_directory = writer.Directory;
if (Verbose)
{
Message("now merge");
Message(" index: " + writer.SegString());
}
// First, quickly run through the newly proposed merges
// and add any orthogonal merges (ie a merge not
// involving segments already pending to be merged) to
// the queue. If we are way behind on merging, many of
// these newly proposed merges will likely already be
// registered.
// Iterate, pulling from the IndexWriter's queue of
// pending merges, until it's empty:
while (true)
{
long startStallTime = 0;
while (writer.HasPendingMerges() && MergeThreadCount >= MaxMergeCount)
{
// this means merging has fallen too far behind: we
// have already created maxMergeCount threads, and
// now there's at least one more merge pending.
// Note that only maxThreadCount of
// those created merge threads will actually be
// running; the rest will be paused (see
// updateMergeThreads). We stall this producer
// thread to prevent creation of new segments,
// until merging has caught up:
startStallTime = Environment.TickCount;
if (Verbose)
{
Message(" too many merges; stalling...");
}
_manualResetEvent.Reset();
_manualResetEvent.Wait();
}
if (Verbose)
{
if (startStallTime != 0)
{
Message(" stalled for " + (Environment.TickCount - startStallTime) + " msec");
}
}
MergePolicy.OneMerge merge = writer.NextMerge();
if (merge is null)
{
if (Verbose)
{
Message(" no more merges pending; now return");
}
return;
}
bool success = false;
try
{
if (Verbose)
{
Message(" consider merge " + writer.SegString(merge.Segments));
}
// OK to spawn a new merge thread to handle this
// merge:
var merger = CreateTask(writer, merge);
merger.MergeThreadCompleted += OnMergeThreadCompleted;
_mergeThreads.Add(merger);
if (Verbose)
{
Message(" launch new thread [" + merger.Name + "]");
}
merger.Start(_taskScheduler);
// Must call this after starting the thread else
// the new thread is removed from mergeThreads
// (since it's not alive yet):
UpdateMergeThreads();
success = true;
}
finally
{
if (!success)
{
writer.MergeFinish(merge);
}
}
}
}
}