public void SpinBackgroundWorkers()
{
workContext.StartWork();
indexingBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew(
new IndexingExecuter(TransactionalStorage, workContext, backgroundTaskScheduler).Execute,
CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler);
tasksBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew(
new TasksExecuter(TransactionalStorage, workContext).Execute,
CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler);
}
private void ExecuteAllInterleaved(IList<IndexingBatchForIndex> result, Action<IndexingBatchForIndex> action)
{
if (result.Count == 0)
return;
/*
This is EXPLICILTY not here, we always want to allow this to run additional indexes
if we still have free spots to run them from threading perspective.
if (result.Count == 1)
{
action(result[0]);
return;
}
*/
var maxNumberOfParallelIndexTasks = context.Configuration.MaxNumberOfParallelIndexTasks;
SortResultsMixedAccordingToTimePerDoc(result);
int isSlowIndex = 0;
var totalIndexingTime = Stopwatch.StartNew();
var tasks = new System.Threading.Tasks.Task[result.Count];
for (int i = 0; i < result.Count; i++)
{
var index = result[i];
var indexToWorkOn = index;
var sp = Stopwatch.StartNew();
var task = new System.Threading.Tasks.Task(() => action(indexToWorkOn));
indexToWorkOn.Index.CurrentMapIndexingTask = tasks[i] = task.ContinueWith(done =>
{
try
{
sp.Stop();
if (done.IsFaulted) // this observe the exception
{
Log.WarnException("Failed to execute indexing task", done.Exception);
}
indexToWorkOn.Index.LastIndexingDuration = sp.Elapsed;
indexToWorkOn.Index.TimePerDoc = sp.ElapsedMilliseconds / Math.Max(1, indexToWorkOn.Batch.Docs.Count);
indexToWorkOn.Index.CurrentMapIndexingTask = null;
return done;
}
catch (ObjectDisposedException)
{
// nothing to do here, this may happen if the database is disposed
// while we have a long running task that didn't complete in time
return new CompletedTask();
}
finally
{
if (indexingSemaphore != null)
indexingSemaphore.Release();
if (indexingCompletedEvent != null)
indexingCompletedEvent.Set();
if (Thread.VolatileRead(ref isSlowIndex) != 0)
{
// we now need to notify the engine that the slow index(es) is done, and we need to resume its indexing
try
{
context.ShouldNotifyAboutWork(() => "Slow Index Completed Indexing Batch");
context.NotifyAboutWork();
}
catch (ObjectDisposedException)
{
// nothing to do here, this may happen if the database is disposed
// while we have a long running task
}
}
}
}).Unwrap();
indexingSemaphore.Wait();
task.Start(context.Database.BackgroundTaskScheduler);
}
// we only get here AFTER we finished scheduling all the indexes
// we wait until we have at least parallel / 2 spots opened (for the _next_ indexing batch) or 8, if we are
// running on Enterprise / high end systems
int minIndexingSpots = Math.Min((maxNumberOfParallelIndexTasks / 2), 8);
while (indexingSemaphore.CurrentCount < minIndexingSpots)
{
indexingCompletedEvent.Wait();
indexingCompletedEvent.Reset();
}
// now we have the chance to start a new indexing batch with the old items, but we still
// want to wait for a bit to _avoid_ creating multiple batches if we can possibly avoid it.
// We will wait for 3/4 the time we waited so far, and a min of 15 seconds
var timeToWait = Math.Max((int)(totalIndexingTime.ElapsedMilliseconds / 4) * 3, 15000);
var totalWaitTime = Stopwatch.StartNew();
while (indexingSemaphore.CurrentCount < maxNumberOfParallelIndexTasks)
{
int timeout = timeToWait - (int)totalWaitTime.ElapsedMilliseconds;
if (timeout <= 0)
break;
indexingCompletedEvent.Reset();
indexingCompletedEvent.Wait(timeout);
}
var creatingNewBatch = indexingSemaphore.CurrentCount < maxNumberOfParallelIndexTasks;
if (creatingNewBatch == false)
return;
Interlocked.Increment(ref isSlowIndex);
if (Log.IsDebugEnabled == false)
return;
var slowIndexes = result.Where(x =>
{
var currentMapIndexingTask = x.Index.CurrentMapIndexingTask;
return currentMapIndexingTask != null && !currentMapIndexingTask.IsCompleted;
})
.Select(x => x.IndexName)
.ToArray();
Log.Debug("Indexing is now split because there are {0:#,#} slow indexes [{1}], memory usage may increase, and those indexing may experience longer stale times (but other indexes will be faster)",
slowIndexes.Length,
string.Join(", ", slowIndexes));
}
private void ExecuteAllInterleaved(IList <IndexingBatchForIndex> result, Action <IndexingBatchForIndex> action)
{
if (result.Count == 0)
{
return;
}
/*
* This is EXPLICILTY not here, we always want to allow this to run additional indexes
* if we still have free spots to run them from threading perspective.
*
* if (result.Count == 1)
* {
* action(result[0]);
* return;
* }
*/
var maxNumberOfParallelIndexTasks = context.Configuration.MaxNumberOfParallelIndexTasks;
SortResultsMixedAccordingToTimePerDoc(result);
int isSlowIndex = 0;
var totalIndexingTime = Stopwatch.StartNew();
var tasks = new System.Threading.Tasks.Task[result.Count];
for (int i = 0; i < result.Count; i++)
{
var index = result[i];
var indexToWorkOn = index;
var sp = Stopwatch.StartNew();
var task = new System.Threading.Tasks.Task(() => action(indexToWorkOn));
indexToWorkOn.Index.CurrentMapIndexingTask = tasks[i] = task.ContinueWith(done =>
{
try
{
sp.Stop();
if (done.IsFaulted) // this observe the exception
{
Log.WarnException("Failed to execute indexing task", done.Exception);
}
indexToWorkOn.Index.LastIndexingDuration = sp.Elapsed;
indexToWorkOn.Index.TimePerDoc = sp.ElapsedMilliseconds / Math.Max(1, indexToWorkOn.Batch.Docs.Count);
indexToWorkOn.Index.CurrentMapIndexingTask = null;
return(done);
}
catch (ObjectDisposedException)
{
// nothing to do here, this may happen if the database is disposed
// while we have a long running task that didn't complete in time
return(new CompletedTask());
}
finally
{
if (indexingSemaphore != null)
{
indexingSemaphore.Release();
}
if (indexingCompletedEvent != null)
{
indexingCompletedEvent.Set();
}
if (Thread.VolatileRead(ref isSlowIndex) != 0)
{
// we now need to notify the engine that the slow index(es) is done, and we need to resume its indexing
try
{
context.ShouldNotifyAboutWork(() => "Slow Index Completed Indexing Batch");
context.NotifyAboutWork();
}
catch (ObjectDisposedException)
{
// nothing to do here, this may happen if the database is disposed
// while we have a long running task
}
}
}
}).Unwrap();
indexingSemaphore.Wait();
task.Start(context.Database.BackgroundTaskScheduler);
}
// we only get here AFTER we finished scheduling all the indexes
// we wait until we have at least parallel / 2 spots opened (for the _next_ indexing batch) or 8, if we are
// running on Enterprise / high end systems
int minIndexingSpots = Math.Min((maxNumberOfParallelIndexTasks / 2), 8);
while (indexingSemaphore.CurrentCount < minIndexingSpots)
{
indexingCompletedEvent.Wait();
indexingCompletedEvent.Reset();
}
// now we have the chance to start a new indexing batch with the old items, but we still
// want to wait for a bit to _avoid_ creating multiple batches if we can possibly avoid it.
// We will wait for 3/4 the time we waited so far, and a min of 15 seconds
var timeToWait = Math.Max((int)(totalIndexingTime.ElapsedMilliseconds / 4) * 3, 15000);
var totalWaitTime = Stopwatch.StartNew();
while (indexingSemaphore.CurrentCount < maxNumberOfParallelIndexTasks)
{
int timeout = timeToWait - (int)totalWaitTime.ElapsedMilliseconds;
if (timeout <= 0)
{
break;
}
indexingCompletedEvent.Reset();
indexingCompletedEvent.Wait(timeout);
}
var creatingNewBatch = indexingSemaphore.CurrentCount < maxNumberOfParallelIndexTasks;
if (creatingNewBatch == false)
{
return;
}
Interlocked.Increment(ref isSlowIndex);
if (Log.IsDebugEnabled == false)
{
return;
}
var slowIndexes = result.Where(x =>
{
var currentMapIndexingTask = x.Index.CurrentMapIndexingTask;
return(currentMapIndexingTask != null && !currentMapIndexingTask.IsCompleted);
})
.Select(x => x.IndexName)
.ToArray();
Log.Debug("Indexing is now split because there are {0:#,#} slow indexes [{1}], memory usage may increase, and those indexing may experience longer stale times (but other indexes will be faster)",
slowIndexes.Length,
string.Join(", ", slowIndexes));
}
public void SpinBackgroundWorkers()
{
workContext.StartWork();
indexingBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew(
new IndexingExecuter(TransactionalStorage, workContext, backgroundTaskScheduler).Execute,
CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler);
reducingBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew(
new ReducingExecuter(TransactionalStorage, workContext, backgroundTaskScheduler).Execute,
CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler);
tasksBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew(
new TasksExecuter(TransactionalStorage, workContext).Execute,
CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler);
}
public void SpinBackgroundWorkers()
{
workContext.StartWork();
backgroundWorkerTask = new System.Threading.Tasks.Task(
new TaskExecuter(TransactionalStorage, workContext).Execute,
TaskCreationOptions.LongRunning);
backgroundWorkerTask.Start();
}
