public bool Execute(QueryOperationContext queryContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token) { var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false ? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan : TimeSpan.FromMinutes(15); var moreWorkFound = false; var totalProcessedCount = 0; foreach (var collection in _index.Collections) { using (var collectionStats = stats.For("Collection_" + collection)) { var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); if (_logger.IsInfoEnabled) { _logger.Info($"Executing map for '{_index.Name}'. Collection: {collection} LastMappedEtag: {lastMappedEtag:#,#;;0}."); } var inMemoryStats = _index.GetStats(collection); var lastEtag = lastMappedEtag; var resultsCount = 0; var pageSize = int.MaxValue; var sw = new Stopwatch(); IndexWriteOperation indexWriter = null; var keepRunning = true; var lastCollectionEtag = -1L; while (keepRunning) { using (queryContext.OpenReadTransaction()) { sw.Restart(); if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastItemEtagInCollection(queryContext, collection); } var items = GetItemsEnumerator(queryContext, collection, lastEtag, pageSize); using (var itemEnumerator = _index.GetMapEnumerator(items, collection, indexContext, collectionStats, _index.Type)) { while (true) { if (itemEnumerator.MoveNext(queryContext.Documents, out IEnumerable mapResults, out var etag) == false) { if (etag > lastEtag) { lastEtag = etag.Value; } collectionStats.RecordMapCompletedReason("No more documents to index"); keepRunning = false; break; } token.ThrowIfCancellationRequested(); if (indexWriter == null) { indexWriter = writeOperation.Value; } var current = itemEnumerator.Current; totalProcessedCount++; collectionStats.RecordMapAttempt(); stats.RecordDocumentSize(current.Size); if (_logger.IsInfoEnabled && totalProcessedCount % 8192 == 0) { _logger.Info($"Executing map for '{_index.Name}'. Processed count: {totalProcessedCount:#,#;;0} etag: {lastEtag:#,#;;0}."); } lastEtag = current.Etag; inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: false); try { var numberOfResults = _index.HandleMap(current, mapResults, indexWriter, indexContext, collectionStats); resultsCount += numberOfResults; collectionStats.RecordMapSuccess(); _index.MapsPerSec?.MarkSingleThreaded(numberOfResults); } catch (Exception e) when(e.IsIndexError()) { itemEnumerator.OnError(); _index.ErrorIndexIfCriticalException(e); collectionStats.RecordMapError(); if (_logger.IsInfoEnabled) { _logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e); } collectionStats.AddMapError(current.Id, $"Failed to execute mapping function on {current.Id}. " + $"Exception: {e}"); } if (CanContinueBatch(queryContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, totalProcessedCount) == false) { keepRunning = false; break; } if (totalProcessedCount >= pageSize) { keepRunning = false; break; } if (MaybeRenewTransaction(queryContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen)) { break; } } } } } if (lastMappedEtag == lastEtag) { // the last mapped etag hasn't changed continue; } moreWorkFound = true; if (_logger.IsInfoEnabled) { _logger.Info($"Executed map for '{_index.Name}' index and '{collection}' collection. Got {resultsCount:#,#;;0} map results in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms."); } if (_index.Type.IsMap()) { _index.SaveLastState(); _indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag); } else { _mapReduceContext.ProcessedDocEtags[collection] = lastEtag; } } } return(moreWorkFound); }
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token) { var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false ? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan : TimeSpan.FromMinutes(15); var moreWorkFound = false; foreach (var collection in _index.Collections) { using (var collectionStats = stats.For("Collection_" + collection)) { if (_logger.IsInfoEnabled) { _logger.Info($"Executing map for '{_index.Name} ({_index.IndexId})'. Collection: {collection}."); } var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); if (_logger.IsInfoEnabled) { _logger.Info($"Executing map for '{_index.Name} ({_index.IndexId})'. LastMappedEtag: {lastMappedEtag}."); } var lastEtag = lastMappedEtag; var count = 0; var resultsCount = 0; var pageSize = int.MaxValue; var sw = new Stopwatch(); IndexWriteOperation indexWriter = null; var keepRunning = true; var lastCollectionEtag = -1L; while (keepRunning) { using (databaseContext.OpenReadTransaction()) { sw.Restart(); if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection); } var documents = GetDocumentsEnumerator(databaseContext, collection, lastEtag, pageSize); using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats)) { while (true) { IEnumerable mapResults; if (docsEnumerator.MoveNext(out mapResults) == false) { collectionStats.RecordMapCompletedReason("No more documents to index"); keepRunning = false; break; } token.ThrowIfCancellationRequested(); if (indexWriter == null) { indexWriter = writeOperation.Value; } var current = docsEnumerator.Current; if (_logger.IsInfoEnabled) { _logger.Info( $"Executing map for '{_index.Name} ({_index.IndexId})'. Processing document: {current.Key}."); } collectionStats.RecordMapAttempt(); count++; lastEtag = current.Etag; try { var numberOfResults = _index.HandleMap(current.LoweredKey, mapResults, indexWriter, indexContext, collectionStats); _index.MapsPerSec.Mark(numberOfResults); resultsCount += numberOfResults; collectionStats.RecordMapSuccess(); } catch (Exception e) { _index.HandleError(e); collectionStats.RecordMapError(); if (_logger.IsInfoEnabled) { _logger.Info( $"Failed to execute mapping function on '{current.Key}' for '{_index.Name} ({_index.IndexId})'.", e); } collectionStats.AddMapError(current.Key, $"Failed to execute mapping function on {current.Key}. Exception: {e}"); } if (CanContinueBatch(collectionStats, lastEtag, lastCollectionEtag) == false) { keepRunning = false; break; } if (count >= pageSize) { keepRunning = false; break; } if (MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen)) { break; } } } } } if (count == 0) { continue; } if (_logger.IsInfoEnabled) { _logger.Info($"Executing map for '{_index.Name} ({_index.IndexId})'. Processed {count:#,#;;0} documents and {resultsCount:#,#;;0} map results in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms."); } if (_index.Type.IsMap()) { _indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag); } else { _mapReduceContext.ProcessedDocEtags[collection] = lastEtag; } moreWorkFound = true; } } return(moreWorkFound); }