protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexesToWorkOn) { indexesToWorkOn = context.Configuration.IndexingScheduler.FilterMapIndexes(indexesToWorkOn); var lastIndexedEtagForAllIndexes = indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToEtag(); context.CancellationToken.ThrowIfCancellationRequested(); var operationCancelled = false; TimeSpan indexingDuration = TimeSpan.Zero; var lastEtag = Etag.Empty; indexesToWorkOn.ForEach(x => x.Index.IsMapIndexingInProgress = true); List <JsonDocument> jsonDocs; using (prefetchingBehavior.DocumentBatchFrom(lastIndexedEtagForAllIndexes, out jsonDocs)) { try { if (Log.IsDebugEnabled) { Log.Debug("Found a total of {0} documents that requires indexing since etag: {1}: ({2})", jsonDocs.Count, lastIndexedEtagForAllIndexes, string.Join(", ", jsonDocs.Select(x => x.Key))); } context.ReportIndexingBatchStarted(jsonDocs.Count, jsonDocs.Sum(x => x.SerializedSizeOnDisk)); context.CancellationToken.ThrowIfCancellationRequested(); if (jsonDocs.Count <= 0) { return; } var sw = Stopwatch.StartNew(); lastEtag = DoActualIndexing(indexesToWorkOn, jsonDocs); indexingDuration = sw.Elapsed; } catch (OperationCanceledException) { operationCancelled = true; } finally { if (operationCancelled == false && jsonDocs != null && jsonDocs.Count > 0) { prefetchingBehavior.CleanupDocuments(lastEtag); prefetchingBehavior.UpdateAutoThrottler(jsonDocs, indexingDuration); } prefetchingBehavior.BatchProcessingComplete(); indexesToWorkOn.ForEach(x => x.Index.IsMapIndexingInProgress = false); } } }
protected override void ExecuteIndexingWork(IList <IndexToWorkOn> indexesToWorkOn) { indexesToWorkOn = context.Configuration.IndexingScheduler.FilterMapIndexes(indexesToWorkOn); if (indexesToWorkOn.Count == 0) { return; } var lastIndexedEtagForAllIndexes = indexesToWorkOn.Min(x => new ComparableByteArray(x.LastIndexedEtag.ToByteArray())).ToEtag(); context.CancellationToken.ThrowIfCancellationRequested(); var operationCancelled = false; TimeSpan indexingDuration = TimeSpan.Zero; List <JsonDocument> jsonDocs = null; var lastEtag = Etag.Empty; indexesToWorkOn.ForEach(x => x.Index.IsMapIndexingInProgress = true); try { jsonDocs = prefetchingBehavior.GetDocumentsBatchFrom(lastIndexedEtagForAllIndexes); if (Log.IsDebugEnabled) { Log.Debug("Found a total of {0} documents that requires indexing since etag: {1}: ({2})", jsonDocs.Count, lastIndexedEtagForAllIndexes, string.Join(", ", jsonDocs.Select(x => x.Key))); } context.ReportIndexingActualBatchSize(jsonDocs.Count); context.CancellationToken.ThrowIfCancellationRequested(); if (jsonDocs.Count <= 0) { return; } var sw = Stopwatch.StartNew(); lastEtag = DoActualIndexing(indexesToWorkOn, jsonDocs); indexingDuration = sw.Elapsed; } catch (InvalidDataException e) { Log.ErrorException("Failed to index because of data corruption. ", e); indexesToWorkOn.ForEach(index => context.AddError(index.IndexId, index.Index.PublicName, null, string.Format("Failed to index because of data corruption. Reason: {0}", e.Message))); } catch (OperationCanceledException) { operationCancelled = true; } finally { if (operationCancelled == false && jsonDocs != null && jsonDocs.Count > 0) { prefetchingBehavior.CleanupDocuments(lastEtag); prefetchingBehavior.UpdateAutoThrottler(jsonDocs, indexingDuration); } prefetchingBehavior.BatchProcessingComplete(); indexesToWorkOn.ForEach(x => x.Index.IsMapIndexingInProgress = false); } }