private Etag DoActualIndexing(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs, IndexingBatchInfo indexingBatchInfo)
{
var lastByEtag = PrefetchingBehavior.GetHighestJsonDocumentByEtag(jsonDocs);
var lastModified = lastByEtag.LastModified.Value;
var lastEtag = lastByEtag.Etag;
context.MetricsCounters.IndexedPerSecond.Mark(jsonDocs.Count);
var result = FilterIndexes(indexesToWorkOn, jsonDocs, lastEtag).OrderByDescending(x => x.Index.LastQueryTime).ToList();
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, result,
index =>
{
using (LogContext.WithDatabase(context.DatabaseName))
{
var performance = HandleIndexingFor(index, lastEtag, lastModified, CancellationToken.None);
if (performance != null)
{
indexingBatchInfo.PerformanceStats.TryAdd(index.Index.PublicName, performance);
}
}
});
return(lastEtag);
}
private Etag DoActualIndexing(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs)
{
var lastByEtag = PrefetchingBehavior.GetHighestJsonDocumentByEtag(jsonDocs);
var lastModified = lastByEtag.LastModified.Value;
var lastEtag = lastByEtag.Etag;
context.IndexedPerSecIncreaseBy(jsonDocs.Count);
var result = FilterIndexes(indexesToWorkOn, jsonDocs, lastEtag).ToList();
ExecuteAllInterleaved(result, index => HandleIndexingFor(index, lastEtag, lastModified));
return(lastEtag);
}
private Etag DoActualIndexing(IList <IndexToWorkOn> indexesToWorkOn, List <JsonDocument> jsonDocs)
{
var lastByEtag = PrefetchingBehavior.GetHighestJsonDocumentByEtag(jsonDocs);
var lastModified = lastByEtag.LastModified.Value;
var lastEtag = lastByEtag.Etag;
context.MetricsCounters.IndexedPerSecond.Mark(jsonDocs.Count);
var result = FilterIndexes(indexesToWorkOn, jsonDocs, lastEtag).ToList();
BackgroundTaskExecuter.Instance.ExecuteAllInterleaved(context, result,
index => HandleIndexingFor(index, lastEtag, lastModified));
return(lastEtag);
}
private bool GenerateIndexingBatchesAndPrefetchDocuments(List<IndexingGroup> groupedIndexes, ConcurrentDictionary<IndexingBatchOperation, object> indexBatchOperations)
{
bool operationWasCancelled = false;
context.Database.MappingThreadPool.ExecuteBatch(groupedIndexes,
indexingGroup =>
{
bool operationAdded = false;
try
{
indexingGroup.PrefetchDocuments();
var curGroupJsonDocs = indexingGroup.JsonDocs;
if (Log.IsDebugEnabled)
{
Log.Debug("Found a total of {0} documents that requires indexing since etag: {1}: ({2})",
curGroupJsonDocs.Count, indexingGroup.LastIndexedEtag, string.Join(", ", curGroupJsonDocs.Select(x => x.Key)));
}
indexingGroup.BatchInfo =
context.ReportIndexingBatchStarted(curGroupJsonDocs.Count,
curGroupJsonDocs.Sum(x => x.SerializedSizeOnDisk),
indexingGroup.Indexes.Select(x => x.Index.PublicName).ToList());
context.CancellationToken.ThrowIfCancellationRequested();
var lastByEtag = PrefetchingBehavior.GetHighestJsonDocumentByEtag(curGroupJsonDocs);
var lastModified = lastByEtag.LastModified.Value;
var lastEtag = lastByEtag.Etag;
List<IndexToWorkOn> filteredOutIndexes;
var indexBatches = FilterIndexes(indexingGroup.Indexes, curGroupJsonDocs, lastEtag, out filteredOutIndexes).OrderByDescending(x => x.Index.LastQueryTime).ToList();
foreach (var filteredOutIndex in filteredOutIndexes)
{
indexingGroup.SignalIndexingComplete();
filteredOutIndex.Index.IsMapIndexingInProgress = false;
}
foreach (var indexBatch in indexBatches)
{
var indexingBatchOperation = new IndexingBatchOperation
{
IndexingBatch = indexBatch,
LastEtag = lastEtag,
LastModified = lastModified,
IndexingBatchInfo = indexingGroup.BatchInfo
};
if (indexBatchOperations.TryAdd(indexingBatchOperation, new object()))
{
indexingBatchOperation.IndexingBatch.OnIndexingComplete += indexingGroup.SignalIndexingComplete;
operationAdded = true;
}
}
}
catch (OperationCanceledException)
{
operationWasCancelled = true;
}
catch (InvalidDataException e)
{
Log.ErrorException("Failed to index because of data corruption. ", e);
indexingGroup.Indexes.ForEach(index =>
context.AddError(index.IndexId, index.Index.PublicName, null, e, string.Format("Failed to index because of data corruption. Reason: {0}", e.Message)));
}
finally
{
if (operationAdded == false)
{
indexingGroup.Indexes.ForEach(x =>
{
indexingGroup.SignalIndexingComplete();
x.Index.IsMapIndexingInProgress = false;
});
}
}
}, description: string.Format("Prefetching Index Groups for {0} groups", groupedIndexes.Count));
return operationWasCancelled;
}