示例#1
0
        private unsafe bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token)
        {
            var moreWorkFound = false;
            Dictionary <string, long> lastIndexedEtagsByCollection = null;

            foreach (var collection in _index.Collections)
            {
                if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false)
                {
                    continue;
                }

                if (lastIndexedEtagsByCollection == null)
                {
                    lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase);
                }

                if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false)
                {
                    lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
                }

                if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now
                {
                    continue;
                }

                foreach (var referencedCollection in referencedCollections)
                {
                    using (var collectionStats = stats.For("Collection_" + referencedCollection.Name))
                    {
                        if (_logger.IsInfoEnabled)
                        {
                            _logger.Info($"Executing handle references for '{_index.Name}'. Collection: {referencedCollection.Name}. Type: {actionType}.");
                        }

                        long lastReferenceEtag;

                        switch (actionType)
                        {
                        case ActionType.Document:
                            lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction, collection, referencedCollection);
                            break;

                        case ActionType.Tombstone:
                            lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection);
                            break;

                        default:
                            throw new NotSupportedException();
                        }

                        if (_logger.IsInfoEnabled)
                        {
                            _logger.Info($"Executing handle references for '{_index.Name}'. LastReferenceEtag: {lastReferenceEtag}.");
                        }

                        var lastEtag = lastReferenceEtag;
                        var count    = 0;

                        var sw = new Stopwatch();
                        IndexWriteOperation indexWriter = null;

                        var keepRunning        = true;
                        var lastCollectionEtag = -1L;
                        while (keepRunning)
                        {
                            var batchCount = 0;

                            using (databaseContext.OpenReadTransaction())
                            {
                                sw.Restart();

                                IEnumerable <Reference> references;
                                switch (actionType)
                                {
                                case ActionType.Document:
                                    if (lastCollectionEtag == -1)
                                    {
                                        lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
                                    }

                                    references = _documentsStorage
                                                 .GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
                                                 .Select(document =>
                                    {
                                        _reference.Key  = document.Id;
                                        _reference.Etag = document.Etag;

                                        return(_reference);
                                    });
                                    break;

                                case ActionType.Tombstone:
                                    if (lastCollectionEtag == -1)
                                    {
                                        lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
                                    }

                                    references = _documentsStorage
                                                 .GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
                                                 .Select(tombstone =>
                                    {
                                        _reference.Key  = tombstone.LowerId;
                                        _reference.Etag = tombstone.Etag;

                                        return(_reference);
                                    });
                                    break;

                                default:
                                    throw new NotSupportedException();
                                }

                                foreach (var referencedDocument in references)
                                {
                                    if (_logger.IsInfoEnabled)
                                    {
                                        _logger.Info($"Executing handle references for '{_index.Name}'. Processing reference: {referencedDocument.Key}.");
                                    }

                                    lastEtag = referencedDocument.Etag;
                                    count++;
                                    batchCount++;

                                    var documents = new List <Document>();
                                    foreach (var key in _indexStorage
                                             .GetDocumentKeysFromCollectionThatReference(collection, referencedDocument.Key, indexContext.Transaction))
                                    {
                                        using (DocumentIdWorker.GetLower(databaseContext.Allocator, key.Content.Ptr, key.Size, out var loweredKey))
                                        {
                                            // when there is conflict, we need to apply same behavior as if the document would not exist
                                            var doc = _documentsStorage.Get(databaseContext, loweredKey, throwOnConflict: false);

                                            if (doc != null && doc.Etag <= lastIndexedEtag)
                                            {
                                                documents.Add(doc);
                                            }
                                        }
                                    }

                                    using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats))
                                    {
                                        while (docsEnumerator.MoveNext(out IEnumerable mapResults))
                                        {
                                            token.ThrowIfCancellationRequested();

                                            var current = docsEnumerator.Current;

                                            if (indexWriter == null)
                                            {
                                                indexWriter = writeOperation.Value;
                                            }

                                            if (_logger.IsInfoEnabled)
                                            {
                                                _logger.Info($"Executing handle references for '{_index.Name}'. Processing document: {current.Id}.");
                                            }

                                            try
                                            {
                                                _index.HandleMap(current.LowerId, mapResults, indexWriter, indexContext, collectionStats);
                                            }
                                            catch (Exception e)
                                            {
                                                if (_logger.IsInfoEnabled)
                                                {
                                                    _logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
                                                }
                                            }
                                        }
                                    }

                                    if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false)
                                    {
                                        keepRunning = false;
                                        break;
                                    }

                                    if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
                                    {
                                        break;
                                    }
                                }

                                if (batchCount == 0 || batchCount >= pageSize)
                                {
                                    break;
                                }
                            }
                        }

                        if (count == 0)
                        {
                            continue;
                        }

                        if (_logger.IsInfoEnabled)
                        {
                            _logger.Info($"Executing handle references for '{_index} ({_index.Name})'. Processed {count} references in '{referencedCollection.Name}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
                        }

                        switch (actionType)
                        {
                        case ActionType.Document:
                            _indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
                            break;

                        case ActionType.Tombstone:
                            _indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
                            break;

                        default:
                            throw new NotSupportedException();
                        }

                        moreWorkFound = true;
                    }
                }
            }

            return(moreWorkFound);
        }
示例#2
0
        public virtual bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext,
                                    Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
        {
            const int pageSize = int.MaxValue;
            var       maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
                ? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
                : TimeSpan.FromMinutes(15);

            var moreWorkFound = false;

            foreach (var collection in _index.Collections)
            {
                using (var collectionStats = stats.For("Collection_" + collection))
                {
                    var lastMappedEtag    = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
                    var lastTombstoneEtag = _indexStorage.ReadLastProcessedTombstoneEtag(indexContext.Transaction, collection);

                    if (_logger.IsInfoEnabled)
                    {
                        _logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Collection: {collection}. LastMappedEtag: {lastMappedEtag:#,#;;0}. LastTombstoneEtag: {lastTombstoneEtag:#,#;;0}.");
                    }

                    var inMemoryStats = _index.GetStats(collection);
                    var lastEtag      = lastTombstoneEtag;
                    var count         = 0;

                    var sw = new Stopwatch();
                    IndexWriteOperation indexWriter = null;
                    var keepRunning        = true;
                    var lastCollectionEtag = -1L;
                    while (keepRunning)
                    {
                        var batchCount = 0;

                        using (databaseContext.OpenReadTransaction())
                        {
                            sw.Restart();

                            if (lastCollectionEtag == -1)
                            {
                                lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
                            }

                            var tombstones = collection == Constants.Documents.Collections.AllDocumentsCollection
                                ? _documentsStorage.GetTombstonesFrom(databaseContext, lastEtag + 1, 0, pageSize)
                                : _documentsStorage.GetTombstonesFrom(databaseContext, collection, lastEtag + 1, 0, pageSize);

                            foreach (var tombstone in tombstones)
                            {
                                token.ThrowIfCancellationRequested();

                                if (indexWriter == null)
                                {
                                    indexWriter = writeOperation.Value;
                                }

                                count++;
                                batchCount++;
                                lastEtag = tombstone.Etag;
                                inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: true);

                                if (_logger.IsInfoEnabled && count % 2048 == 0)
                                {
                                    _logger.Info($"Executing cleanup for '{_index.Name}'. Processed count: {count:#,#;;0} etag: {lastEtag}.");
                                }

                                if (tombstone.Type != Tombstone.TombstoneType.Document)
                                {
                                    continue; // this can happen when we have '@all_docs'
                                }
                                _index.HandleDelete(tombstone, collection, indexWriter, indexContext, collectionStats);

                                if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false)
                                {
                                    keepRunning = false;
                                    break;
                                }

                                if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
                                {
                                    break;
                                }
                            }

                            if (batchCount == 0 || batchCount >= pageSize)
                            {
                                break;
                            }
                        }
                    }

                    if (count == 0)
                    {
                        continue;
                    }

                    if (_logger.IsInfoEnabled)
                    {
                        _logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processed {count} tombstones in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
                    }

                    if (_index.Type.IsMap())
                    {
                        _indexStorage.WriteLastTombstoneEtag(indexContext.Transaction, collection, lastEtag);
                    }
                    else
                    {
                        _mapReduceContext.ProcessedTombstoneEtags[collection] = lastEtag;
                    }

                    moreWorkFound = true;
                }
            }

            return(moreWorkFound);
        }
示例#3
0
        private bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token)
        {
            var moreWorkFound = false;
            Dictionary <string, long> lastIndexedEtagsByCollection = null;

            foreach (var collection in _index.Collections)
            {
                if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false)
                {
                    continue;
                }

                if (lastIndexedEtagsByCollection == null)
                {
                    lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase);
                }

                if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false)
                {
                    lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
                }

                if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now
                {
                    continue;
                }

                var totalProcessedCount = 0;

                foreach (var referencedCollection in referencedCollections)
                {
                    var inMemoryStats = _index.GetReferencesStats(referencedCollection.Name);

                    using (var collectionStats = stats.For("Collection_" + referencedCollection.Name))
                    {
                        long lastReferenceEtag;

                        switch (actionType)
                        {
                        case ActionType.Document:
                            lastReferenceEtag = IndexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
                            break;

                        case ActionType.Tombstone:
                            lastReferenceEtag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
                            break;

                        default:
                            throw new NotSupportedException();
                        }

                        var lastEtag     = lastReferenceEtag;
                        var resultsCount = 0;

                        var sw = new Stopwatch();

                        var keepRunning        = true;
                        var lastCollectionEtag = -1L;
                        while (keepRunning)
                        {
                            var hasChanges = false;

                            using (databaseContext.OpenReadTransaction())
                            {
                                sw.Restart();

                                IEnumerable <Reference> references;
                                switch (actionType)
                                {
                                case ActionType.Document:
                                    if (lastCollectionEtag == -1)
                                    {
                                        lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
                                    }

                                    references = _documentsStorage
                                                 .GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize,
                                                                   DocumentFields.Id | DocumentFields.Etag)
                                                 .Select(document =>
                                    {
                                        _reference.Key  = document.Id;
                                        _reference.Etag = document.Etag;

                                        return(_reference);
                                    });
                                    break;

                                case ActionType.Tombstone:
                                    if (lastCollectionEtag == -1)
                                    {
                                        lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
                                    }

                                    references = _documentsStorage
                                                 .GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
                                                 .Select(tombstone =>
                                    {
                                        _reference.Key  = tombstone.LowerId;
                                        _reference.Etag = tombstone.Etag;

                                        return(_reference);
                                    });
                                    break;

                                default:
                                    throw new NotSupportedException();
                                }

                                var isTombstone = actionType == ActionType.Tombstone;

                                foreach (var referencedDocument in references)
                                {
                                    lastEtag   = referencedDocument.Etag;
                                    hasChanges = true;
                                    inMemoryStats.UpdateLastEtag(lastEtag, isTombstone);

                                    var documents = GetDocumentFromCollectionThatReference(databaseContext, indexContext, collection, referencedDocument, lastIndexedEtag);

                                    using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats, _index.Type))
                                    {
                                        while (docsEnumerator.MoveNext(out IEnumerable mapResults))
                                        {
                                            token.ThrowIfCancellationRequested();

                                            totalProcessedCount++;
                                            collectionStats.RecordMapReferenceAttempt();

                                            var current = docsEnumerator.Current;
                                            stats.RecordDocumentSize(current.Data.Size);

                                            try
                                            {
                                                var numberOfResults = _index.HandleMap(current.LowerId, current.Id, mapResults, writeOperation, indexContext, collectionStats);

                                                resultsCount += numberOfResults;
                                                collectionStats.RecordMapReferenceSuccess();
                                                _index.MapsPerSec.MarkSingleThreaded(numberOfResults);
                                            }
                                            catch (Exception e) when(e.IsIndexError())
                                            {
                                                docsEnumerator.OnError();
                                                _index.ErrorIndexIfCriticalException(e);

                                                collectionStats.RecordMapReferenceError();
                                                if (_logger.IsInfoEnabled)
                                                {
                                                    _logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
                                                }

                                                collectionStats.AddMapReferenceError(current.Id,
                                                                                     $"Failed to execute mapping function on {current.Id}. Exception: {e}");
                                            }

                                            _index.UpdateThreadAllocations(indexContext, writeOperation, stats, updateReduceStats: false);
                                        }
                                    }

                                    if (CanContinueBatch(databaseContext, indexContext, collectionStats, writeOperation, lastEtag, lastCollectionEtag, totalProcessedCount) == false)
                                    {
                                        keepRunning = false;
                                        break;
                                    }

                                    if (totalProcessedCount >= pageSize)
                                    {
                                        keepRunning = false;
                                        break;
                                    }

                                    if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
                                    {
                                        break;
                                    }
                                }

                                if (hasChanges == false)
                                {
                                    break;
                                }
                            }
                        }

                        if (lastReferenceEtag == lastEtag)
                        {
                            // the last referenced etag hasn't changed
                            continue;
                        }

                        moreWorkFound = true;

                        if (_logger.IsInfoEnabled)
                        {
                            _logger.Info($"Executed handle references for '{_index.Name}' index and '{referencedCollection.Name}' collection. " +
                                         $"Got {resultsCount:#,#;;0} map results in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
                        }

                        switch (actionType)
                        {
                        case ActionType.Document:
                            _indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
                            break;

                        case ActionType.Tombstone:
                            _indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
                            break;

                        default:
                            throw new NotSupportedException();
                        }
                    }
                }
            }

            return(moreWorkFound);
        }