public void ReportIndexingBatchStarted(int documentsCount, long documentsSize)
{
var indexingBatchInfo = new IndexingBatchInfo
{
TotalDocumentCount = documentsCount,
TotalDocumentSize = documentsSize,
Timestamp = SystemTime.UtcNow
};
lastActualIndexingBatchInfo.Add(indexingBatchInfo);
}
public void ReportIndexingBatchStarted(int documentsCount, long documentsSize, List <string> indexesToWorkOn, out IndexingBatchInfo indexingBatchInfo)
{
indexingBatchInfo = new IndexingBatchInfo
{
IndexesToWorkOn = indexesToWorkOn,
TotalDocumentCount = documentsCount,
TotalDocumentSize = documentsSize,
StartedAt = SystemTime.UtcNow,
PerformanceStats = new ConcurrentDictionary <string, IndexingPerformanceStats>()
};
lastActualIndexingBatchInfo.Add(indexingBatchInfo);
}
public void IndexPrecomputedBatch(PrecomputedIndexingBatch precomputedBatch, CancellationToken token)
{
token.ThrowIfCancellationRequested();
context.MetricsCounters.IndexedPerSecond.Mark(precomputedBatch.Documents.Count);
var indexToWorkOn = new IndexToWorkOn
{
Index = precomputedBatch.Index,
IndexId = precomputedBatch.Index.indexId,
LastIndexedEtag = Etag.Empty
};
using (LogContext.WithDatabase(context.DatabaseName))
using (MapIndexingInProgress(new List <IndexToWorkOn> {
indexToWorkOn
}))
{
var indexingBatchForIndex =
FilterIndexes(new List <IndexToWorkOn> {
indexToWorkOn
}, precomputedBatch.Documents,
precomputedBatch.LastIndexed).FirstOrDefault();
if (indexingBatchForIndex == null)
{
return;
}
IndexingBatchInfo batchInfo = null;
IndexingPerformanceStats performance = null;
try
{
batchInfo = context.ReportIndexingBatchStarted(precomputedBatch.Documents.Count, -1, new List <string>
{
indexToWorkOn.Index.PublicName
});
batchInfo.BatchType = BatchType.Precomputed;
if (Log.IsDebugEnabled)
{
Log.Debug("Going to index precomputed documents for a new index {0}. Count of precomputed docs {1}",
precomputedBatch.Index.PublicName, precomputedBatch.Documents.Count);
}
performance = HandleIndexingFor(indexingBatchForIndex, precomputedBatch.LastIndexed, precomputedBatch.LastModified, token);
}
finally
{
if (batchInfo != null)
{
if (performance != null)
{
batchInfo.PerformanceStats.TryAdd(indexingBatchForIndex.Index.PublicName, performance);
}
context.ReportIndexingBatchCompleted(batchInfo);
}
}
}
indexReplacer.ReplaceIndexes(new [] { indexToWorkOn.IndexId });
}
private Etag DoActualIndexing(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs, IndexingBatchInfo indexingBatchInfo)
{
var lastByEtag = PrefetchingBehavior.GetHighestJsonDocumentByEtag(jsonDocs);
var lastModified = lastByEtag.LastModified.Value;
var lastEtag = lastByEtag.Etag;
context.MetricsCounters.IndexedPerSecond.Mark(jsonDocs.Count);
var result = FilterIndexes(indexesToWorkOn, jsonDocs, lastEtag).OrderByDescending(x => x.Index.LastQueryTime).ToList();
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, result,
index =>
{
using (LogContext.WithDatabase(context.DatabaseName))
{
var performance = HandleIndexingFor(index, lastEtag, lastModified, CancellationToken.None);
if (performance != null)
{
indexingBatchInfo.PerformanceStats.TryAdd(index.Index.PublicName, performance);
}
}
});
return(lastEtag);
}
protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexes)
{
var indexingGroups = context.Configuration.IndexingClassifier.GroupMapIndexes(indexes);
indexingGroups = indexingGroups.OrderByDescending(x => x.Key).ToDictionary(x => x.Key, x => x.Value);
if (indexingGroups.Count == 0)
{
return;
}
var usedPrefetchers = new ConcurrentSet <PrefetchingBehavior>();
var groupedIndexes = indexingGroups.Select(x =>
{
var result = new IndexingGroup
{
LastIndexedEtag = x.Key,
Indexes = x.Value,
LastQueryTime = x.Value.Max(y => y.Index.LastQueryTime),
PrefetchingBehavior = GetPrefetcherFor(x.Key, usedPrefetchers)
};
result.PrefetchingBehavior.AdditionalInfo = string.Format("Default prefetcher: {0}. For indexing group: [Indexes: {1}, LastIndexedEtag: {2}]",
result.PrefetchingBehavior == defaultPrefetchingBehavior, string.Join(", ", result.Indexes.Select(y => y.Index.PublicName)), result.LastIndexedEtag);
return(result);
}).OrderByDescending(x => x.LastQueryTime).ToList();
var maxIndexOutputsPerDoc = groupedIndexes.Max(x => x.Indexes.Max(y => y.Index.MaxIndexOutputsPerDocument));
var containsMapReduceIndexes = groupedIndexes.Any(x => x.Indexes.Any(y => y.Index.IsMapReduce));
var recoverTunerState = ((IndexBatchSizeAutoTuner)autoTuner).ConsiderLimitingNumberOfItemsToProcessForThisBatch(maxIndexOutputsPerDoc, containsMapReduceIndexes);
BackgroundTaskExecuter.Instance.ExecuteAll(context, groupedIndexes, (indexingGroup, i) =>
{
context.CancellationToken.ThrowIfCancellationRequested();
using (LogContext.WithDatabase(context.DatabaseName))
{
var prefetchingBehavior = indexingGroup.PrefetchingBehavior;
var indexesToWorkOn = indexingGroup.Indexes;
var operationCancelled = false;
TimeSpan indexingDuration = TimeSpan.Zero;
var lastEtag = Etag.Empty;
List <JsonDocument> jsonDocs;
IndexingBatchInfo batchInfo = null;
using (MapIndexingInProgress(indexesToWorkOn))
using (prefetchingBehavior.DocumentBatchFrom(indexingGroup.LastIndexedEtag, out jsonDocs))
{
try
{
if (Log.IsDebugEnabled)
{
Log.Debug("Found a total of {0} documents that requires indexing since etag: {1}: ({2})",
jsonDocs.Count, indexingGroup.LastIndexedEtag, string.Join(", ", jsonDocs.Select(x => x.Key)));
}
batchInfo = context.ReportIndexingBatchStarted(jsonDocs.Count, jsonDocs.Sum(x => x.SerializedSizeOnDisk), indexesToWorkOn.Select(x => x.Index.PublicName).ToList());
context.CancellationToken.ThrowIfCancellationRequested();
if (jsonDocs.Count <= 0)
{
return;
}
var sw = Stopwatch.StartNew();
lastEtag = DoActualIndexing(indexesToWorkOn, jsonDocs, batchInfo);
indexingDuration = sw.Elapsed;
}
catch (InvalidDataException e)
{
Log.ErrorException("Failed to index because of data corruption. ", e);
indexesToWorkOn.ForEach(index =>
context.AddError(index.IndexId, index.Index.PublicName, null, string.Format("Failed to index because of data corruption. Reason: {0}", e.Message)));
}
catch (OperationCanceledException)
{
operationCancelled = true;
}
finally
{
if (operationCancelled == false && jsonDocs != null && jsonDocs.Count > 0)
{
prefetchingBehavior.CleanupDocuments(lastEtag);
prefetchingBehavior.UpdateAutoThrottler(jsonDocs, indexingDuration);
}
prefetchingBehavior.BatchProcessingComplete();
if (batchInfo != null)
{
context.ReportIndexingBatchCompleted(batchInfo);
}
}
}
}
});
if (recoverTunerState != null)
{
recoverTunerState();
}
RemoveUnusedPrefetchers(usedPrefetchers);
}
public void IndexPrecomputedBatch(PrecomputedIndexingBatch precomputedBatch)
{
context.MetricsCounters.IndexedPerSecond.Mark(precomputedBatch.Documents.Count);
var indexToWorkOn = new IndexToWorkOn()
{
Index = precomputedBatch.Index,
IndexId = precomputedBatch.Index.indexId,
LastIndexedEtag = Etag.Empty
};
using (MapIndexingInProgress(new List <IndexToWorkOn> {
indexToWorkOn
}))
{
var indexingBatchForIndex =
FilterIndexes(new List <IndexToWorkOn>()
{
indexToWorkOn
}, precomputedBatch.Documents,
precomputedBatch.LastIndexed).FirstOrDefault();
if (indexingBatchForIndex == null)
{
return;
}
IndexingBatchInfo batchInfo = null;
try
{
context.ReportIndexingBatchStarted(precomputedBatch.Documents.Count, precomputedBatch.Documents.Sum(x => x.SerializedSizeOnDisk), new List <string>()
{
indexToWorkOn.Index.PublicName
}, out batchInfo);
batchInfo.BatchType = BatchType.Precomputed;
if (Log.IsDebugEnabled)
{
Log.Debug("Going to index precomputed documents for a new index {0}. Count of precomputed docs {1}",
precomputedBatch.Index.PublicName, precomputedBatch.Documents.Count);
}
HandleIndexingFor(indexingBatchForIndex, precomputedBatch.LastIndexed, precomputedBatch.LastModified);
}
finally
{
var performance = indexingBatchForIndex.Batch.GetIndexingPerformance();
if (batchInfo != null)
{
if (performance != null)
{
batchInfo.PerformanceStats.TryAdd(indexingBatchForIndex.Index.PublicName, performance);
}
batchInfo.BatchCompleted();
}
}
}
}
public void ReportIndexingBatchCompleted(IndexingBatchInfo batchInfo)
{
batchInfo.BatchCompleted();
batchInfo.Id = Interlocked.Increment(ref nextIndexingBatchInfoId);
LastActualIndexingBatchInfo.Add(batchInfo);
}
public void ReportIndexingBatchCompleted(IndexingBatchInfo batchInfo)
{
batchInfo.BatchCompleted();
lastActualIndexingBatchInfo.Add(batchInfo);
}
protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexes)
{
var indexingGroups = context.Configuration.IndexingClassifier.GroupMapIndexes(indexes);
indexingGroups = indexingGroups.OrderByDescending(x => x.Key).ToDictionary(x => x.Key, x => x.Value);
if (indexingGroups.Count == 0)
{
return;
}
var usedPrefetchers = new ConcurrentSet <PrefetchingBehavior>();
var groupedIndexes = indexingGroups.Select(x =>
{
var result = new IndexingGroup
{
LastIndexedEtag = x.Key,
Indexes = x.Value,
LastQueryTime = x.Value.Max(y => y.Index.LastQueryTime),
};
SetPrefetcherForIndexingGroup(result, usedPrefetchers);
return(result);
}).OrderByDescending(x => x.LastQueryTime).ToList();
var maxIndexOutputsPerDoc = groupedIndexes.Max(x => x.Indexes.Max(y => y.Index.MaxIndexOutputsPerDocument));
var containsMapReduceIndexes = groupedIndexes.Any(x => x.Indexes.Any(y => y.Index.IsMapReduce));
var recoverTunerState = ((IndexBatchSizeAutoTuner)autoTuner).ConsiderLimitingNumberOfItemsToProcessForThisBatch(maxIndexOutputsPerDoc, containsMapReduceIndexes);
BackgroundTaskExecuter.Instance.ExecuteAll(context, groupedIndexes, (indexingGroup, i) =>
{
context.CancellationToken.ThrowIfCancellationRequested();
using (LogContext.WithDatabase(context.DatabaseName))
{
var prefetchingBehavior = indexingGroup.PrefetchingBehavior;
var indexesToWorkOn = indexingGroup.Indexes;
var operationCanceled = false;
TimeSpan indexingDuration = TimeSpan.Zero;
var lastEtag = Etag.Empty;
IndexingBatchInfo batchInfo = null;
try
{
using (MapIndexingInProgress(indexesToWorkOn))
{
List <JsonDocument> jsonDocs;
using (prefetchingBehavior.DocumentBatchFrom(indexingGroup.LastIndexedEtag, out jsonDocs))
{
try
{
if (Log.IsDebugEnabled)
{
Log.Debug("Found a total of {0} documents that requires indexing since etag: {1}: ({2})",
jsonDocs.Count, indexingGroup.LastIndexedEtag, string.Join(", ", jsonDocs.Select(x => x.Key)));
}
batchInfo = context.ReportIndexingBatchStarted(jsonDocs.Count, jsonDocs.Sum(x => x.SerializedSizeOnDisk), indexesToWorkOn.Select(x => x.Index.PublicName).ToList());
context.CancellationToken.ThrowIfCancellationRequested();
if (jsonDocs.Count <= 0)
{
return;
}
var sw = Stopwatch.StartNew();
lastEtag = DoActualIndexing(indexesToWorkOn, jsonDocs, batchInfo);
indexingDuration = sw.Elapsed;
}
catch (InvalidDataException e)
{
Log.ErrorException("Failed to index because of data corruption. ", e);
indexesToWorkOn.ForEach(index =>
context.AddError(index.IndexId, index.Index.PublicName, null, string.Format("Failed to index because of data corruption. Reason: {0}", e.Message)));
}
catch (OperationCanceledException)
{
operationCanceled = true;
}
catch (AggregateException e)
{
var allOperationsCanceled = e
.InnerExceptions
.All(x => x is OperationCanceledException);
if (allOperationsCanceled == false)
{
throw;
}
operationCanceled = true;
}
catch (Exception)
{
//we should not clean docs from prefetching behavior because something
//unexpected has thrown
//logging will be done in catch in the outer scope
operationCanceled = true;
// ReSharper disable once ThrowingSystemException
throw;
}
finally
{
if (operationCanceled == false && jsonDocs != null && jsonDocs.Count > 0)
{
prefetchingBehavior.CleanupDocuments(lastEtag);
prefetchingBehavior.UpdateAutoThrottler(jsonDocs, indexingDuration);
}
prefetchingBehavior.BatchProcessingComplete();
if (batchInfo != null)
{
context.ReportIndexingBatchCompleted(batchInfo);
}
}
}
}
}
catch (OperationCanceledException)
{
//expected exception here, nothing to do
}
catch (AggregateException e)
{
var allOperationsCanceled = e
.InnerExceptions
.All(x => x is OperationCanceledException);
if (allOperationsCanceled == false)
{
var message = string.Format("Unexpected AggregateException happened during execution of indexing batch...this is not supposed to happen. Reason: {0}", e);
Log.Error(message, e);
indexesToWorkOn.ForEach(index => context.AddError(index.IndexId, index.Index.PublicName, null, message));
}
}
catch (Exception e)
{
//this is a precaution, no exception should happen at this point
var message = string.Format("Unexpected exception happened during execution of indexing batch...this is not supposed to happen. Reason: {0}", e);
Log.Error(message, e);
indexesToWorkOn.ForEach(index => context.AddError(index.IndexId, index.Index.PublicName, null, message));
//rethrow because we do not want to interrupt the existing exception flow
// ReSharper disable once ThrowingSystemException
throw;
}
}
});
if (recoverTunerState != null)
{
recoverTunerState();
}
RemoveUnusedPrefetchers(usedPrefetchers);
}
