private unsafe bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token) { var moreWorkFound = false; Dictionary <string, long> lastIndexedEtagsByCollection = null; foreach (var collection in _index.Collections) { if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false) { continue; } if (lastIndexedEtagsByCollection == null) { lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase); } if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false) { lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); } if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now { continue; } foreach (var referencedCollection in referencedCollections) { using (var collectionStats = stats.For("Collection_" + referencedCollection.Name)) { if (_logger.IsInfoEnabled) { _logger.Info($"Executing handle references for '{_index.Name}'. Collection: {referencedCollection.Name}. Type: {actionType}."); } long lastReferenceEtag; switch (actionType) { case ActionType.Document: lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction, collection, referencedCollection); break; case ActionType.Tombstone: lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection); break; default: throw new NotSupportedException(); } if (_logger.IsInfoEnabled) { _logger.Info($"Executing handle references for '{_index.Name}'. LastReferenceEtag: {lastReferenceEtag}."); } var lastEtag = lastReferenceEtag; var count = 0; var sw = new Stopwatch(); IndexWriteOperation indexWriter = null; var keepRunning = true; var lastCollectionEtag = -1L; while (keepRunning) { var batchCount = 0; using (databaseContext.OpenReadTransaction()) { sw.Restart(); IEnumerable <Reference> references; switch (actionType) { case ActionType.Document: if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection); } references = _documentsStorage .GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize) .Select(document => { _reference.Key = document.Id; _reference.Etag = document.Etag; return(_reference); }); break; case ActionType.Tombstone: if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection); } references = _documentsStorage .GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize) .Select(tombstone => { _reference.Key = tombstone.LowerId; _reference.Etag = tombstone.Etag; return(_reference); }); break; default: throw new NotSupportedException(); } foreach (var referencedDocument in references) { if (_logger.IsInfoEnabled) { _logger.Info($"Executing handle references for '{_index.Name}'. Processing reference: {referencedDocument.Key}."); } lastEtag = referencedDocument.Etag; count++; batchCount++; var documents = new List <Document>(); foreach (var key in _indexStorage .GetDocumentKeysFromCollectionThatReference(collection, referencedDocument.Key, indexContext.Transaction)) { using (DocumentIdWorker.GetLower(databaseContext.Allocator, key.Content.Ptr, key.Size, out var loweredKey)) { // when there is conflict, we need to apply same behavior as if the document would not exist var doc = _documentsStorage.Get(databaseContext, loweredKey, throwOnConflict: false); if (doc != null && doc.Etag <= lastIndexedEtag) { documents.Add(doc); } } } using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats)) { while (docsEnumerator.MoveNext(out IEnumerable mapResults)) { token.ThrowIfCancellationRequested(); var current = docsEnumerator.Current; if (indexWriter == null) { indexWriter = writeOperation.Value; } if (_logger.IsInfoEnabled) { _logger.Info($"Executing handle references for '{_index.Name}'. Processing document: {current.Id}."); } try { _index.HandleMap(current.LowerId, mapResults, indexWriter, indexContext, collectionStats); } catch (Exception e) { if (_logger.IsInfoEnabled) { _logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e); } } } } if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false) { keepRunning = false; break; } if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen)) { break; } } if (batchCount == 0 || batchCount >= pageSize) { break; } } } if (count == 0) { continue; } if (_logger.IsInfoEnabled) { _logger.Info($"Executing handle references for '{_index} ({_index.Name})'. Processed {count} references in '{referencedCollection.Name}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms."); } switch (actionType) { case ActionType.Document: _indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag); break; case ActionType.Tombstone: _indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag); break; default: throw new NotSupportedException(); } moreWorkFound = true; } } } return(moreWorkFound); }
public virtual bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token) { const int pageSize = int.MaxValue; var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false ? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan : TimeSpan.FromMinutes(15); var moreWorkFound = false; foreach (var collection in _index.Collections) { using (var collectionStats = stats.For("Collection_" + collection)) { var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); var lastTombstoneEtag = _indexStorage.ReadLastProcessedTombstoneEtag(indexContext.Transaction, collection); if (_logger.IsInfoEnabled) { _logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Collection: {collection}. LastMappedEtag: {lastMappedEtag:#,#;;0}. LastTombstoneEtag: {lastTombstoneEtag:#,#;;0}."); } var inMemoryStats = _index.GetStats(collection); var lastEtag = lastTombstoneEtag; var count = 0; var sw = new Stopwatch(); IndexWriteOperation indexWriter = null; var keepRunning = true; var lastCollectionEtag = -1L; while (keepRunning) { var batchCount = 0; using (databaseContext.OpenReadTransaction()) { sw.Restart(); if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection); } var tombstones = collection == Constants.Documents.Collections.AllDocumentsCollection ? _documentsStorage.GetTombstonesFrom(databaseContext, lastEtag + 1, 0, pageSize) : _documentsStorage.GetTombstonesFrom(databaseContext, collection, lastEtag + 1, 0, pageSize); foreach (var tombstone in tombstones) { token.ThrowIfCancellationRequested(); if (indexWriter == null) { indexWriter = writeOperation.Value; } count++; batchCount++; lastEtag = tombstone.Etag; inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: true); if (_logger.IsInfoEnabled && count % 2048 == 0) { _logger.Info($"Executing cleanup for '{_index.Name}'. Processed count: {count:#,#;;0} etag: {lastEtag}."); } if (tombstone.Type != Tombstone.TombstoneType.Document) { continue; // this can happen when we have '@all_docs' } _index.HandleDelete(tombstone, collection, indexWriter, indexContext, collectionStats); if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false) { keepRunning = false; break; } if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen)) { break; } } if (batchCount == 0 || batchCount >= pageSize) { break; } } } if (count == 0) { continue; } if (_logger.IsInfoEnabled) { _logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processed {count} tombstones in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms."); } if (_index.Type.IsMap()) { _indexStorage.WriteLastTombstoneEtag(indexContext.Transaction, collection, lastEtag); } else { _mapReduceContext.ProcessedTombstoneEtags[collection] = lastEtag; } moreWorkFound = true; } } return(moreWorkFound); }
private bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token) { var moreWorkFound = false; Dictionary <string, long> lastIndexedEtagsByCollection = null; foreach (var collection in _index.Collections) { if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false) { continue; } if (lastIndexedEtagsByCollection == null) { lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase); } if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false) { lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); } if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now { continue; } var totalProcessedCount = 0; foreach (var referencedCollection in referencedCollections) { var inMemoryStats = _index.GetReferencesStats(referencedCollection.Name); using (var collectionStats = stats.For("Collection_" + referencedCollection.Name)) { long lastReferenceEtag; switch (actionType) { case ActionType.Document: lastReferenceEtag = IndexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); break; case ActionType.Tombstone: lastReferenceEtag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); break; default: throw new NotSupportedException(); } var lastEtag = lastReferenceEtag; var resultsCount = 0; var sw = new Stopwatch(); var keepRunning = true; var lastCollectionEtag = -1L; while (keepRunning) { var hasChanges = false; using (databaseContext.OpenReadTransaction()) { sw.Restart(); IEnumerable <Reference> references; switch (actionType) { case ActionType.Document: if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection); } references = _documentsStorage .GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize, DocumentFields.Id | DocumentFields.Etag) .Select(document => { _reference.Key = document.Id; _reference.Etag = document.Etag; return(_reference); }); break; case ActionType.Tombstone: if (lastCollectionEtag == -1) { lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection); } references = _documentsStorage .GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize) .Select(tombstone => { _reference.Key = tombstone.LowerId; _reference.Etag = tombstone.Etag; return(_reference); }); break; default: throw new NotSupportedException(); } var isTombstone = actionType == ActionType.Tombstone; foreach (var referencedDocument in references) { lastEtag = referencedDocument.Etag; hasChanges = true; inMemoryStats.UpdateLastEtag(lastEtag, isTombstone); var documents = GetDocumentFromCollectionThatReference(databaseContext, indexContext, collection, referencedDocument, lastIndexedEtag); using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats, _index.Type)) { while (docsEnumerator.MoveNext(out IEnumerable mapResults)) { token.ThrowIfCancellationRequested(); totalProcessedCount++; collectionStats.RecordMapReferenceAttempt(); var current = docsEnumerator.Current; stats.RecordDocumentSize(current.Data.Size); try { var numberOfResults = _index.HandleMap(current.LowerId, current.Id, mapResults, writeOperation, indexContext, collectionStats); resultsCount += numberOfResults; collectionStats.RecordMapReferenceSuccess(); _index.MapsPerSec.MarkSingleThreaded(numberOfResults); } catch (Exception e) when(e.IsIndexError()) { docsEnumerator.OnError(); _index.ErrorIndexIfCriticalException(e); collectionStats.RecordMapReferenceError(); if (_logger.IsInfoEnabled) { _logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e); } collectionStats.AddMapReferenceError(current.Id, $"Failed to execute mapping function on {current.Id}. Exception: {e}"); } _index.UpdateThreadAllocations(indexContext, writeOperation, stats, updateReduceStats: false); } } if (CanContinueBatch(databaseContext, indexContext, collectionStats, writeOperation, lastEtag, lastCollectionEtag, totalProcessedCount) == false) { keepRunning = false; break; } if (totalProcessedCount >= pageSize) { keepRunning = false; break; } if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen)) { break; } } if (hasChanges == false) { break; } } } if (lastReferenceEtag == lastEtag) { // the last referenced etag hasn't changed continue; } moreWorkFound = true; if (_logger.IsInfoEnabled) { _logger.Info($"Executed handle references for '{_index.Name}' index and '{referencedCollection.Name}' collection. " + $"Got {resultsCount:#,#;;0} map results in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms."); } switch (actionType) { case ActionType.Document: _indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag); break; case ActionType.Tombstone: _indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag); break; default: throw new NotSupportedException(); } } } } return(moreWorkFound); }