protected override void ExecuteIndexingWork(IList <IndexToWorkOn> mapReduceIndexes)
{
ReducingBatchInfo reducingBatchInfo = null;
try
{
var currentlyRunning = currentlyProcessedIndexes.Keys;
//we filter the indexes that are already running
var indexesToWorkOn = mapReduceIndexes.Where(x => currentlyRunning.Contains(x.IndexId) == false).ToList();
reducingBatchInfo = context.ReportReducingBatchStarted(indexesToWorkOn.Select(x => x.Index.PublicName).ToList());
if (context.Database.ThreadPool == null || context.RunReducing == false)
{
throw new OperationCanceledException();
}
var oomeErrorsCount = indexesToWorkOn.Select(x => x.Index).Sum(x => x.OutOfMemoryErrorsCount);
var ranToCompletion =
context.Database.ThreadPool.ExecuteBatch(indexesToWorkOn, indexToWorkOn =>
{
if (currentlyProcessedIndexes.TryAdd(indexToWorkOn.IndexId, indexToWorkOn.Index) == false)
{
//shouldn't happen since we already filtered the running reduce indexes
Log.Warn("Tried to run a map-reduce for index '{0}' that is already running!", GetIndexName(indexToWorkOn.IndexId));
return;
}
try
{
var performanceStats = HandleReduceForIndex(indexToWorkOn, oomeErrorsCount > 0, context.CancellationToken);
if (performanceStats != null)
{
reducingBatchInfo.PerformanceStats.TryAdd(indexToWorkOn.Index.PublicName, performanceStats);
}
indexToWorkOn.Index.DecrementReducingOutOfMemoryErrors();
}
finally
{
Index _;
currentlyProcessedIndexes.TryRemove(indexToWorkOn.IndexId, out _);
}
}, allowPartialBatchResumption: MemoryStatistics.AvailableMemoryInMb > 1.5 * context.Configuration.MemoryLimitForProcessingInMb,
description: $"Executing indexes reduction on {indexesToWorkOn.Count} indexes", database: context.Database, runAfterCompletion: context.NotifyAboutWork);
if (ranToCompletion == false)
{
context.NotifyAboutWork();
}
}
finally
{
if (reducingBatchInfo != null)
{
context.ReportReducingBatchCompleted(reducingBatchInfo);
}
}
}
protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexesToWorkOn)
{
ReducingBatchInfo reducingBatchInfo = null;
int executedPartially = 0;
try
{
reducingBatchInfo = context.ReportReducingBatchStarted(indexesToWorkOn.Select(x => x.Index.PublicName).ToList());
context.Database.ReducingThreadPool.ExecuteBatch(indexesToWorkOn, index =>
{
var performanceStats = HandleReduceForIndex(index, context.CancellationToken);
if (performanceStats != null)
{
reducingBatchInfo.PerformanceStats.TryAdd(index.Index.PublicName, performanceStats);
}
if (Thread.VolatileRead(ref executedPartially) == 1)
{
context.NotifyAboutWork();
}
}, allowPartialBatchResumption: MemoryStatistics.AvailableMemory > 1.5 * context.Configuration.Memory.LimitForProcessing, description: string.Format("Executing Indexex Reduction on {0} indexes", indexesToWorkOn.Count));
Interlocked.Increment(ref executedPartially);
}
finally
{
if (reducingBatchInfo != null)
{
context.ReportReducingBatchCompleted(reducingBatchInfo);
}
}
}
protected override void ExecuteIndexingWork(IList<IndexToWorkOn> indexesToWorkOn)
{
ReducingBatchInfo reducingBatchInfo = null;
try
{
reducingBatchInfo = context.ReportReducingBatchStarted(indexesToWorkOn.Select(x => x.Index.PublicName).ToList());
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, indexesToWorkOn, index =>
{
var performanceStats = HandleReduceForIndex(index, context.CancellationToken);
reducingBatchInfo.PerformanceStats.TryAdd(index.Index.PublicName, performanceStats);
});
}
finally
{
if (reducingBatchInfo != null)
context.ReportReducingBatchCompleted(reducingBatchInfo);
}
}
protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexesToWorkOn)
{
ReducingBatchInfo reducingBatchInfo = null;
try
{
reducingBatchInfo = context.ReportReducingBatchStarted(indexesToWorkOn.Select(x => x.Index.PublicName).ToList());
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, indexesToWorkOn, indexToWorkOn =>
{
if (currentlyProcessedIndexes.TryAdd(indexToWorkOn.IndexId, indexToWorkOn.Index) == false)
{
Log.Warn("Tried to run a map-reduce for index '{0}' that is already running!", GetIndexName(indexToWorkOn.IndexId));
return;
}
try
{
var performanceStats = HandleReduceForIndex(indexToWorkOn, context.CancellationToken);
reducingBatchInfo.PerformanceStats.TryAdd(indexToWorkOn.Index.PublicName, performanceStats);
}
finally
{
Index _;
currentlyProcessedIndexes.TryRemove(indexToWorkOn.IndexId, out _);
}
});
}
finally
{
if (reducingBatchInfo != null)
{
context.ReportReducingBatchCompleted(reducingBatchInfo);
}
}
}
public void ReportReducingBatchCompleted(ReducingBatchInfo batchInfo)
{
batchInfo.BatchCompleted();
batchInfo.Id = Interlocked.Increment(ref nextReducingBatchInfoId);
LastActualReducingBatchInfo.Add(batchInfo);
}
public void ReportReducingBatchCompleted(ReducingBatchInfo batchInfo)
{
batchInfo.BatchCompleted();
lastActualReducingBatchInfo.Add(batchInfo);
}
