public void SpinBackgroundWorkers() { workContext.StartWork(); indexingBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew( new IndexingExecuter(TransactionalStorage, workContext, backgroundTaskScheduler).Execute, CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler); tasksBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew( new TasksExecuter(TransactionalStorage, workContext).Execute, CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler); }
private void ExecuteAllInterleaved(IList<IndexingBatchForIndex> result, Action<IndexingBatchForIndex> action) { if (result.Count == 0) return; /* This is EXPLICILTY not here, we always want to allow this to run additional indexes if we still have free spots to run them from threading perspective. if (result.Count == 1) { action(result[0]); return; } */ var maxNumberOfParallelIndexTasks = context.Configuration.MaxNumberOfParallelIndexTasks; SortResultsMixedAccordingToTimePerDoc(result); int isSlowIndex = 0; var totalIndexingTime = Stopwatch.StartNew(); var tasks = new System.Threading.Tasks.Task[result.Count]; for (int i = 0; i < result.Count; i++) { var index = result[i]; var indexToWorkOn = index; var sp = Stopwatch.StartNew(); var task = new System.Threading.Tasks.Task(() => action(indexToWorkOn)); indexToWorkOn.Index.CurrentMapIndexingTask = tasks[i] = task.ContinueWith(done => { try { sp.Stop(); if (done.IsFaulted) // this observe the exception { Log.WarnException("Failed to execute indexing task", done.Exception); } indexToWorkOn.Index.LastIndexingDuration = sp.Elapsed; indexToWorkOn.Index.TimePerDoc = sp.ElapsedMilliseconds / Math.Max(1, indexToWorkOn.Batch.Docs.Count); indexToWorkOn.Index.CurrentMapIndexingTask = null; return done; } catch (ObjectDisposedException) { // nothing to do here, this may happen if the database is disposed // while we have a long running task that didn't complete in time return new CompletedTask(); } finally { if (indexingSemaphore != null) indexingSemaphore.Release(); if (indexingCompletedEvent != null) indexingCompletedEvent.Set(); if (Thread.VolatileRead(ref isSlowIndex) != 0) { // we now need to notify the engine that the slow index(es) is done, and we need to resume its indexing try { context.ShouldNotifyAboutWork(() => "Slow Index Completed Indexing Batch"); context.NotifyAboutWork(); } catch (ObjectDisposedException) { // nothing to do here, this may happen if the database is disposed // while we have a long running task } } } }).Unwrap(); indexingSemaphore.Wait(); task.Start(context.Database.BackgroundTaskScheduler); } // we only get here AFTER we finished scheduling all the indexes // we wait until we have at least parallel / 2 spots opened (for the _next_ indexing batch) or 8, if we are // running on Enterprise / high end systems int minIndexingSpots = Math.Min((maxNumberOfParallelIndexTasks / 2), 8); while (indexingSemaphore.CurrentCount < minIndexingSpots) { indexingCompletedEvent.Wait(); indexingCompletedEvent.Reset(); } // now we have the chance to start a new indexing batch with the old items, but we still // want to wait for a bit to _avoid_ creating multiple batches if we can possibly avoid it. // We will wait for 3/4 the time we waited so far, and a min of 15 seconds var timeToWait = Math.Max((int)(totalIndexingTime.ElapsedMilliseconds / 4) * 3, 15000); var totalWaitTime = Stopwatch.StartNew(); while (indexingSemaphore.CurrentCount < maxNumberOfParallelIndexTasks) { int timeout = timeToWait - (int)totalWaitTime.ElapsedMilliseconds; if (timeout <= 0) break; indexingCompletedEvent.Reset(); indexingCompletedEvent.Wait(timeout); } var creatingNewBatch = indexingSemaphore.CurrentCount < maxNumberOfParallelIndexTasks; if (creatingNewBatch == false) return; Interlocked.Increment(ref isSlowIndex); if (Log.IsDebugEnabled == false) return; var slowIndexes = result.Where(x => { var currentMapIndexingTask = x.Index.CurrentMapIndexingTask; return currentMapIndexingTask != null && !currentMapIndexingTask.IsCompleted; }) .Select(x => x.IndexName) .ToArray(); Log.Debug("Indexing is now split because there are {0:#,#} slow indexes [{1}], memory usage may increase, and those indexing may experience longer stale times (but other indexes will be faster)", slowIndexes.Length, string.Join(", ", slowIndexes)); }
private void ExecuteAllInterleaved(IList <IndexingBatchForIndex> result, Action <IndexingBatchForIndex> action) { if (result.Count == 0) { return; } /* * This is EXPLICILTY not here, we always want to allow this to run additional indexes * if we still have free spots to run them from threading perspective. * * if (result.Count == 1) * { * action(result[0]); * return; * } */ var maxNumberOfParallelIndexTasks = context.Configuration.MaxNumberOfParallelIndexTasks; SortResultsMixedAccordingToTimePerDoc(result); int isSlowIndex = 0; var totalIndexingTime = Stopwatch.StartNew(); var tasks = new System.Threading.Tasks.Task[result.Count]; for (int i = 0; i < result.Count; i++) { var index = result[i]; var indexToWorkOn = index; var sp = Stopwatch.StartNew(); var task = new System.Threading.Tasks.Task(() => action(indexToWorkOn)); indexToWorkOn.Index.CurrentMapIndexingTask = tasks[i] = task.ContinueWith(done => { try { sp.Stop(); if (done.IsFaulted) // this observe the exception { Log.WarnException("Failed to execute indexing task", done.Exception); } indexToWorkOn.Index.LastIndexingDuration = sp.Elapsed; indexToWorkOn.Index.TimePerDoc = sp.ElapsedMilliseconds / Math.Max(1, indexToWorkOn.Batch.Docs.Count); indexToWorkOn.Index.CurrentMapIndexingTask = null; return(done); } catch (ObjectDisposedException) { // nothing to do here, this may happen if the database is disposed // while we have a long running task that didn't complete in time return(new CompletedTask()); } finally { if (indexingSemaphore != null) { indexingSemaphore.Release(); } if (indexingCompletedEvent != null) { indexingCompletedEvent.Set(); } if (Thread.VolatileRead(ref isSlowIndex) != 0) { // we now need to notify the engine that the slow index(es) is done, and we need to resume its indexing try { context.ShouldNotifyAboutWork(() => "Slow Index Completed Indexing Batch"); context.NotifyAboutWork(); } catch (ObjectDisposedException) { // nothing to do here, this may happen if the database is disposed // while we have a long running task } } } }).Unwrap(); indexingSemaphore.Wait(); task.Start(context.Database.BackgroundTaskScheduler); } // we only get here AFTER we finished scheduling all the indexes // we wait until we have at least parallel / 2 spots opened (for the _next_ indexing batch) or 8, if we are // running on Enterprise / high end systems int minIndexingSpots = Math.Min((maxNumberOfParallelIndexTasks / 2), 8); while (indexingSemaphore.CurrentCount < minIndexingSpots) { indexingCompletedEvent.Wait(); indexingCompletedEvent.Reset(); } // now we have the chance to start a new indexing batch with the old items, but we still // want to wait for a bit to _avoid_ creating multiple batches if we can possibly avoid it. // We will wait for 3/4 the time we waited so far, and a min of 15 seconds var timeToWait = Math.Max((int)(totalIndexingTime.ElapsedMilliseconds / 4) * 3, 15000); var totalWaitTime = Stopwatch.StartNew(); while (indexingSemaphore.CurrentCount < maxNumberOfParallelIndexTasks) { int timeout = timeToWait - (int)totalWaitTime.ElapsedMilliseconds; if (timeout <= 0) { break; } indexingCompletedEvent.Reset(); indexingCompletedEvent.Wait(timeout); } var creatingNewBatch = indexingSemaphore.CurrentCount < maxNumberOfParallelIndexTasks; if (creatingNewBatch == false) { return; } Interlocked.Increment(ref isSlowIndex); if (Log.IsDebugEnabled == false) { return; } var slowIndexes = result.Where(x => { var currentMapIndexingTask = x.Index.CurrentMapIndexingTask; return(currentMapIndexingTask != null && !currentMapIndexingTask.IsCompleted); }) .Select(x => x.IndexName) .ToArray(); Log.Debug("Indexing is now split because there are {0:#,#} slow indexes [{1}], memory usage may increase, and those indexing may experience longer stale times (but other indexes will be faster)", slowIndexes.Length, string.Join(", ", slowIndexes)); }
public void SpinBackgroundWorkers() { workContext.StartWork(); indexingBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew( new IndexingExecuter(TransactionalStorage, workContext, backgroundTaskScheduler).Execute, CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler); reducingBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew( new ReducingExecuter(TransactionalStorage, workContext, backgroundTaskScheduler).Execute, CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler); tasksBackgroundTask = System.Threading.Tasks.Task.Factory.StartNew( new TasksExecuter(TransactionalStorage, workContext).Execute, CancellationToken.None, TaskCreationOptions.LongRunning, backgroundTaskScheduler); }
public void SpinBackgroundWorkers() { workContext.StartWork(); backgroundWorkerTask = new System.Threading.Tasks.Task( new TaskExecuter(TransactionalStorage, workContext).Execute, TaskCreationOptions.LongRunning); backgroundWorkerTask.Start(); }