예제 #1
0
        protected override void ExecuteIndexingWork(IList <IndexToWorkOn> mapReduceIndexes)
        {
            ReducingBatchInfo reducingBatchInfo = null;

            try
            {
                var currentlyRunning = currentlyProcessedIndexes.Keys;
                //we filter the indexes that are already running
                var indexesToWorkOn = mapReduceIndexes.Where(x => currentlyRunning.Contains(x.IndexId) == false).ToList();

                reducingBatchInfo = context.ReportReducingBatchStarted(indexesToWorkOn.Select(x => x.Index.PublicName).ToList());

                if (context.Database.ThreadPool == null || context.RunReducing == false)
                {
                    throw new OperationCanceledException();
                }

                var oomeErrorsCount = indexesToWorkOn.Select(x => x.Index).Sum(x => x.OutOfMemoryErrorsCount);

                var ranToCompletion =
                    context.Database.ThreadPool.ExecuteBatch(indexesToWorkOn, indexToWorkOn =>
                {
                    if (currentlyProcessedIndexes.TryAdd(indexToWorkOn.IndexId, indexToWorkOn.Index) == false)
                    {
                        //shouldn't happen since we already filtered the running reduce indexes
                        Log.Warn("Tried to run a map-reduce for index '{0}' that is already running!", GetIndexName(indexToWorkOn.IndexId));
                        return;
                    }

                    try
                    {
                        var performanceStats = HandleReduceForIndex(indexToWorkOn, oomeErrorsCount > 0, context.CancellationToken);
                        if (performanceStats != null)
                        {
                            reducingBatchInfo.PerformanceStats.TryAdd(indexToWorkOn.Index.PublicName, performanceStats);
                        }

                        indexToWorkOn.Index.DecrementReducingOutOfMemoryErrors();
                    }
                    finally
                    {
                        Index _;
                        currentlyProcessedIndexes.TryRemove(indexToWorkOn.IndexId, out _);
                    }
                }, allowPartialBatchResumption: MemoryStatistics.AvailableMemoryInMb > 1.5 * context.Configuration.MemoryLimitForProcessingInMb,
                                                             description: $"Executing indexes reduction on {indexesToWorkOn.Count} indexes", database: context.Database, runAfterCompletion: context.NotifyAboutWork);

                if (ranToCompletion == false)
                {
                    context.NotifyAboutWork();
                }
            }
            finally
            {
                if (reducingBatchInfo != null)
                {
                    context.ReportReducingBatchCompleted(reducingBatchInfo);
                }
            }
        }
예제 #2
0
        protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexesToWorkOn)
        {
            ReducingBatchInfo reducingBatchInfo = null;

            int executedPartially = 0;

            try
            {
                reducingBatchInfo = context.ReportReducingBatchStarted(indexesToWorkOn.Select(x => x.Index.PublicName).ToList());

                context.Database.ReducingThreadPool.ExecuteBatch(indexesToWorkOn, index =>
                {
                    var performanceStats = HandleReduceForIndex(index, context.CancellationToken);

                    if (performanceStats != null)
                    {
                        reducingBatchInfo.PerformanceStats.TryAdd(index.Index.PublicName, performanceStats);
                    }

                    if (Thread.VolatileRead(ref executedPartially) == 1)
                    {
                        context.NotifyAboutWork();
                    }
                }, allowPartialBatchResumption: MemoryStatistics.AvailableMemory > 1.5 * context.Configuration.Memory.LimitForProcessing, description: string.Format("Executing Indexex Reduction on {0} indexes", indexesToWorkOn.Count));
                Interlocked.Increment(ref executedPartially);
            }
            finally
            {
                if (reducingBatchInfo != null)
                {
                    context.ReportReducingBatchCompleted(reducingBatchInfo);
                }
            }
        }
예제 #3
0
		protected override void ExecuteIndexingWork(IList<IndexToWorkOn> indexesToWorkOn)
		{
			ReducingBatchInfo reducingBatchInfo = null;

			try
			{
				reducingBatchInfo = context.ReportReducingBatchStarted(indexesToWorkOn.Select(x => x.Index.PublicName).ToList());

				BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, indexesToWorkOn, index =>
				{
					var performanceStats = HandleReduceForIndex(index, context.CancellationToken);

					reducingBatchInfo.PerformanceStats.TryAdd(index.Index.PublicName, performanceStats);
				});
			}
			finally
			{
				if (reducingBatchInfo != null)
					context.ReportReducingBatchCompleted(reducingBatchInfo);
			}
		}
예제 #4
0
        protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexesToWorkOn)
        {
            ReducingBatchInfo reducingBatchInfo = null;

            try
            {
                reducingBatchInfo = context.ReportReducingBatchStarted(indexesToWorkOn.Select(x => x.Index.PublicName).ToList());

                BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, indexesToWorkOn, indexToWorkOn =>
                {
                    if (currentlyProcessedIndexes.TryAdd(indexToWorkOn.IndexId, indexToWorkOn.Index) == false)
                    {
                        Log.Warn("Tried to run a map-reduce for index '{0}' that is already running!", GetIndexName(indexToWorkOn.IndexId));
                        return;
                    }

                    try
                    {
                        var performanceStats = HandleReduceForIndex(indexToWorkOn, context.CancellationToken);

                        reducingBatchInfo.PerformanceStats.TryAdd(indexToWorkOn.Index.PublicName, performanceStats);
                    }
                    finally
                    {
                        Index _;
                        currentlyProcessedIndexes.TryRemove(indexToWorkOn.IndexId, out _);
                    }
                });
            }
            finally
            {
                if (reducingBatchInfo != null)
                {
                    context.ReportReducingBatchCompleted(reducingBatchInfo);
                }
            }
        }
예제 #5
0
 public void ReportReducingBatchCompleted(ReducingBatchInfo batchInfo)
 {
     batchInfo.BatchCompleted();
     batchInfo.Id = Interlocked.Increment(ref nextReducingBatchInfoId);
     LastActualReducingBatchInfo.Add(batchInfo);
 }
예제 #6
0
 public void ReportReducingBatchCompleted(ReducingBatchInfo batchInfo)
 {
     batchInfo.BatchCompleted();
     lastActualReducingBatchInfo.Add(batchInfo);
 }