protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexesToWorkOn)
{
indexesToWorkOn = context.Configuration.IndexingScheduler.FilterMapIndexes(indexesToWorkOn);
var lastIndexedEtagForAllIndexes =
indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToEtag();
context.CancellationToken.ThrowIfCancellationRequested();
var operationCancelled = false;
TimeSpan indexingDuration = TimeSpan.Zero;
var lastEtag = Etag.Empty;
indexesToWorkOn.ForEach(x => x.Index.IsMapIndexingInProgress = true);
List <JsonDocument> jsonDocs;
using (prefetchingBehavior.DocumentBatchFrom(lastIndexedEtagForAllIndexes, out jsonDocs))
{
try
{
if (Log.IsDebugEnabled)
{
Log.Debug("Found a total of {0} documents that requires indexing since etag: {1}: ({2})",
jsonDocs.Count, lastIndexedEtagForAllIndexes, string.Join(", ", jsonDocs.Select(x => x.Key)));
}
context.ReportIndexingBatchStarted(jsonDocs.Count, jsonDocs.Sum(x => x.SerializedSizeOnDisk));
context.CancellationToken.ThrowIfCancellationRequested();
if (jsonDocs.Count <= 0)
{
return;
}
var sw = Stopwatch.StartNew();
lastEtag = DoActualIndexing(indexesToWorkOn, jsonDocs);
indexingDuration = sw.Elapsed;
}
catch (OperationCanceledException)
{
operationCancelled = true;
}
finally
{
if (operationCancelled == false && jsonDocs != null && jsonDocs.Count > 0)
{
prefetchingBehavior.CleanupDocuments(lastEtag);
prefetchingBehavior.UpdateAutoThrottler(jsonDocs, indexingDuration);
}
prefetchingBehavior.BatchProcessingComplete();
indexesToWorkOn.ForEach(x => x.Index.IsMapIndexingInProgress = false);
}
}
}
protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexesToWorkOn)
{
indexesToWorkOn = context.Configuration.IndexingScheduler.FilterMapIndexes(indexesToWorkOn);
if (indexesToWorkOn.Count == 0)
{
return;
}
var lastIndexedEtagForAllIndexes =
indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToEtag();
context.CancellationToken.ThrowIfCancellationRequested();
var operationCancelled = false;
TimeSpan indexingDuration = TimeSpan.Zero;
List <JsonDocument> jsonDocs = null;
var lastEtag = Etag.Empty;
indexesToWorkOn.ForEach(x => x.Index.IsMapIndexingInProgress = true);
try
{
jsonDocs = prefetchingBehavior.GetDocumentsBatchFrom(lastIndexedEtagForAllIndexes);
if (Log.IsDebugEnabled)
{
Log.Debug("Found a total of {0} documents that requires indexing since etag: {1}: ({2})",
jsonDocs.Count, lastIndexedEtagForAllIndexes, string.Join(", ", jsonDocs.Select(x => x.Key)));
}
context.ReportIndexingActualBatchSize(jsonDocs.Count);
context.CancellationToken.ThrowIfCancellationRequested();
if (jsonDocs.Count <= 0)
{
return;
}
var sw = Stopwatch.StartNew();
lastEtag = DoActualIndexing(indexesToWorkOn, jsonDocs);
indexingDuration = sw.Elapsed;
}
catch (InvalidDataException e)
{
Log.ErrorException("Failed to index because of data corruption. ", e);
indexesToWorkOn.ForEach(index =>
context.AddError(index.IndexId, index.Index.PublicName, null, string.Format("Failed to index because of data corruption. Reason: {0}", e.Message)));
}
catch (OperationCanceledException)
{
operationCancelled = true;
}
finally
{
if (operationCancelled == false && jsonDocs != null && jsonDocs.Count > 0)
{
prefetchingBehavior.CleanupDocuments(lastEtag);
prefetchingBehavior.UpdateAutoThrottler(jsonDocs, indexingDuration);
}
prefetchingBehavior.BatchProcessingComplete();
indexesToWorkOn.ForEach(x => x.Index.IsMapIndexingInProgress = false);
}
}