Exemple #1
0
        private long InitializeLastDatabaseEtagOnIndexCreation(TransactionOperationContext indexContext)
        {
            const string key = "LastEtag";

            if (_environment.IsNew == false)
            {
                var tree   = indexContext.Transaction.InnerTransaction.ReadTree(IndexSchema.LastDocumentEtagOnIndexCreationTree);
                var result = tree?.Read(key);
                return(result?.Reader.ReadLittleEndianInt64() ?? 0);
            }

            using (var queryContext = QueryOperationContext.Allocate(DocumentDatabase, _index))
                using (queryContext.OpenReadTransaction())
                    using (Slice.From(indexContext.Allocator, key, out var slice))
                    {
                        var lastDatabaseEtag = DocumentsStorage.ReadLastEtag(queryContext.Documents.Transaction.InnerTransaction);
                        var tree             = indexContext.Transaction.InnerTransaction.CreateTree(IndexSchema.LastDocumentEtagOnIndexCreationTree);
                        tree.Add(slice, lastDatabaseEtag);

                        return(lastDatabaseEtag);
                    }
        }
Exemple #2
0
        public static bool ShouldReplace(Index index, ref bool?isSideBySide)
        {
            if (isSideBySide.HasValue == false)
            {
                isSideBySide = index.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase);
            }

            if (isSideBySide == false)
            {
                return(false);
            }

            using (var context = QueryOperationContext.Allocate(index.DocumentDatabase, index))
                using (index._contextPool.AllocateOperationContext(out TransactionOperationContext indexContext))
                {
                    using (indexContext.OpenReadTransaction())
                        using (context.OpenReadTransaction())
                        {
                            indexContext.IgnoreStalenessDueToReduceOutputsToDelete = true;

                            try
                            {
                                var canReplace = index.IsStale(context, indexContext) == false;
                                if (canReplace)
                                {
                                    isSideBySide = null;
                                }

                                return(canReplace);
                            }
                            finally
                            {
                                indexContext.IgnoreStalenessDueToReduceOutputsToDelete = false;
                            }
                        }
                }
        }
Exemple #3
0
        private static bool IsStaleDueToReferences(Index index, AbstractStaticIndexBase compiled, QueryOperationContext queryContext, TransactionOperationContext indexContext, long?referenceCutoff, long?compareExchangeReferenceCutoff, List <string> stalenessReasons)
        {
            foreach (var collection in index.Collections)
            {
                long lastIndexedEtag = -1;

                if (compiled.ReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections))
                {
                    lastIndexedEtag = index._indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);

                    // we haven't handled references for that collection yet
                    // in theory we could check what is the last etag for that collection in documents store
                    // but this was checked earlier by the base index class
                    if (lastIndexedEtag > 0)
                    {
                        foreach (var referencedCollection in referencedCollections)
                        {
                            var lastDocEtag = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(queryContext.Documents.Transaction.InnerTransaction, referencedCollection.Name);
                            var lastProcessedReferenceEtag = index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
                            var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);

                            if (referenceCutoff == null)
                            {
                                if (lastDocEtag > lastProcessedReferenceEtag)
                                {
                                    if (stalenessReasons == null)
                                    {
                                        return(true);
                                    }

                                    var lastDoc = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetByEtag(queryContext.Documents, lastDocEtag);

                                    stalenessReasons.Add($"There are still some document references to process from collection '{referencedCollection.Name}'. " +
                                                         $"The last document etag in that collection is '{lastDocEtag:#,#;;0}' " +
                                                         $"({Constants.Documents.Metadata.Id}: '{lastDoc.Id}', " +
                                                         $"{Constants.Documents.Metadata.LastModified}: '{lastDoc.LastModified}'), " +
                                                         $"but last processed document etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'.");
                                }

                                var lastTombstoneEtag = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetLastTombstoneEtag(queryContext.Documents.Transaction.InnerTransaction, referencedCollection.Name);

                                if (lastTombstoneEtag > lastProcessedTombstoneEtag)
                                {
                                    if (stalenessReasons == null)
                                    {
                                        return(true);
                                    }

                                    var lastTombstone = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetTombstoneByEtag(queryContext.Documents, lastTombstoneEtag);

                                    stalenessReasons.Add($"There are still some tombstone references to process from collection '{referencedCollection.Name}'. " +
                                                         $"The last tombstone etag in that collection is '{lastTombstoneEtag:#,#;;0}' " +
                                                         $"({Constants.Documents.Metadata.Id}: '{lastTombstone.LowerId}', " +
                                                         $"{Constants.Documents.Metadata.LastModified}: '{lastTombstone.LastModified}'), " +
                                                         $"but last processed tombstone etag for that collection is '{lastProcessedTombstoneEtag:#,#;;0}'.");
                                }
                            }
                            else
                            {
                                var minDocEtag = Math.Min(referenceCutoff.Value, lastDocEtag);
                                if (minDocEtag > lastProcessedReferenceEtag)
                                {
                                    if (stalenessReasons == null)
                                    {
                                        return(true);
                                    }

                                    var lastDoc = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetByEtag(queryContext.Documents, lastDocEtag);

                                    stalenessReasons.Add($"There are still some document references to process from collection '{referencedCollection.Name}'. " +
                                                         $"The last document etag in that collection is '{lastDocEtag:#,#;;0}' " +
                                                         $"({Constants.Documents.Metadata.Id}: '{lastDoc.Id}', " +
                                                         $"{Constants.Documents.Metadata.LastModified}: '{lastDoc.LastModified}') " +
                                                         $"with cutoff set to '{referenceCutoff.Value}', " +
                                                         $"but last processed document etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'.");
                                }

                                var hasTombstones = queryContext.Documents.DocumentDatabase.DocumentsStorage.HasTombstonesWithEtagGreaterThanStartAndLowerThanOrEqualToEnd(queryContext.Documents, referencedCollection.Name,
                                                                                                                                                                           lastProcessedTombstoneEtag,
                                                                                                                                                                           referenceCutoff.Value);

                                if (hasTombstones)
                                {
                                    if (stalenessReasons == null)
                                    {
                                        return(true);
                                    }

                                    stalenessReasons.Add($"There are still some tombstones to process from collection '{referencedCollection.Name}' with etag range '{lastProcessedTombstoneEtag} - {referenceCutoff.Value}'.");
                                }
                            }
                        }
                    }
                }

                if (compiled.CollectionsWithCompareExchangeReferences.Contains(collection))
                {
                    if (lastIndexedEtag == -1)
                    {
                        lastIndexedEtag = index._indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
                    }

                    // we haven't handled references for that collection yet
                    // in theory we could check what is the last etag for that collection in documents store
                    // but this was checked earlier by the base index class
                    if (lastIndexedEtag > 0)
                    {
                        var lastCompareExchangeEtag    = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.GetLastCompareExchangeIndexForDatabase(queryContext.Server, queryContext.Documents.DocumentDatabase.Name);
                        var lastProcessedReferenceEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange);

                        if (compareExchangeReferenceCutoff == null)
                        {
                            if (lastCompareExchangeEtag > lastProcessedReferenceEtag)
                            {
                                if (stalenessReasons == null)
                                {
                                    return(true);
                                }

                                stalenessReasons.Add($"There are still some compare exchange references to process for collection '{collection}'. The last compare exchange etag is '{lastCompareExchangeEtag:#,#;;0}', but last processed compare exchange etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'.");
                            }

                            var lastTombstoneEtag          = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.GetLastCompareExchangeTombstoneIndexForDatabase(queryContext.Server, queryContext.Documents.DocumentDatabase.Name);
                            var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange);

                            if (lastTombstoneEtag > lastProcessedTombstoneEtag)
                            {
                                if (stalenessReasons == null)
                                {
                                    return(true);
                                }

                                stalenessReasons.Add($"There are still some compare exchange tombstone references to process for collection '{collection}'. The last compare exchange tombstone etag is '{lastTombstoneEtag:#,#;;0}', but last processed compare exchange tombstone etag for that collection is '{lastProcessedTombstoneEtag:#,#;;0}'.");
                            }
                        }
                        else
                        {
                            var minCompareExchangeEtag = Math.Min(compareExchangeReferenceCutoff.Value, lastCompareExchangeEtag);
                            if (minCompareExchangeEtag > lastProcessedReferenceEtag)
                            {
                                if (stalenessReasons == null)
                                {
                                    return(true);
                                }

                                stalenessReasons.Add($"There are still some compare exchange references to process for collection '{collection}'. The last compare exchange etag is '{lastCompareExchangeEtag:#,#;;0}' with cutoff set to '{compareExchangeReferenceCutoff.Value}', but last processed compare exchange etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'.");
                            }

                            var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange);
                            var hasTombstones = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.HasCompareExchangeTombstonesWithEtagGreaterThanStartAndLowerThanOrEqualToEnd(queryContext.Server, queryContext.Documents.DocumentDatabase.Name,
                                                                                                                                                                                         lastProcessedTombstoneEtag,
                                                                                                                                                                                         compareExchangeReferenceCutoff.Value);

                            if (hasTombstones)
                            {
                                if (stalenessReasons == null)
                                {
                                    return(true);
                                }

                                stalenessReasons.Add($"There are still some compare exchange tombstones to process for collection '{collection}' with etag range '{lastProcessedTombstoneEtag} - {compareExchangeReferenceCutoff.Value}'.");
                            }
                        }
                    }
                }
            }

            return(stalenessReasons?.Count > 0);
        }
Exemple #4
0
 public static bool IsStaleDueToReferences(MapReduceIndex index, QueryOperationContext queryContext, TransactionOperationContext indexContext, long?referenceCutoff, long?compareExchangeReferenceCutoff, List <string> stalenessReasons)
 {
     return(IsStaleDueToReferences(index, index._compiled, queryContext, indexContext, referenceCutoff, compareExchangeReferenceCutoff, stalenessReasons));
 }
Exemple #5
0
        public static unsafe long CalculateIndexEtag(Index index, AbstractStaticIndexBase compiled, int length, byte *indexEtagBytes, byte *writePos, QueryOperationContext queryContext, TransactionOperationContext indexContext)
        {
            foreach (var collection in index.Collections)
            {
                if (compiled.ReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections))
                {
                    foreach (var referencedCollection in referencedCollections)
                    {
                        var lastDocEtag = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(queryContext.Documents.Transaction.InnerTransaction, referencedCollection.Name);
                        var lastProcessedReferenceEtag = index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);

                        var lastTombstoneEtag          = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetLastTombstoneEtag(queryContext.Documents.Transaction.InnerTransaction, referencedCollection.Name);
                        var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);

                        *(long *)writePos = lastDocEtag;
                        writePos         += sizeof(long);
                        *(long *)writePos = lastProcessedReferenceEtag;
                        writePos         += sizeof(long);
                        *(long *)writePos = lastTombstoneEtag;
                        writePos         += sizeof(long);
                        *(long *)writePos = lastProcessedTombstoneEtag;

                        var referencesInfo = index.GetInMemoryReferencesState(collection, isCompareExchange: false);
                        writePos         += sizeof(long);
                        *(long *)writePos = referencesInfo.ParentItemEtag;
                        writePos         += sizeof(long);
                        *(long *)writePos = referencesInfo.ParentTombstoneEtag;
                    }
                }

                if (compiled.CollectionsWithCompareExchangeReferences.Contains(collection))
                {
                    var lastCompareExchangeEtag    = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.GetLastCompareExchangeIndexForDatabase(queryContext.Server, queryContext.Documents.DocumentDatabase.Name);
                    var lastProcessedReferenceEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange);

                    var lastTombstoneEtag          = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.GetLastCompareExchangeTombstoneIndexForDatabase(queryContext.Server, queryContext.Documents.DocumentDatabase.Name);
                    var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange);

                    *(long *)writePos = lastCompareExchangeEtag;
                    writePos         += sizeof(long);
                    *(long *)writePos = lastProcessedReferenceEtag;
                    writePos         += sizeof(long);
                    *(long *)writePos = lastTombstoneEtag;
                    writePos         += sizeof(long);
                    *(long *)writePos = lastProcessedTombstoneEtag;

                    var referencesInfo = index.GetInMemoryReferencesState(collection, isCompareExchange: true);
                    writePos         += sizeof(long);
                    *(long *)writePos = referencesInfo.ParentItemEtag;
                    writePos         += sizeof(long);
                    *(long *)writePos = referencesInfo.ParentTombstoneEtag;
                }
            }

            unchecked
            {
                return((long)Hashing.XXHash64.Calculate(indexEtagBytes, (ulong)length));
            }
        }
Exemple #6
0
 public static unsafe long CalculateIndexEtag(MapReduceIndex index, int length, byte *indexEtagBytes, byte *writePos, QueryOperationContext queryContext, TransactionOperationContext indexContext)
 {
     return(CalculateIndexEtag(index, index._compiled, length, indexEtagBytes, writePos, queryContext, indexContext));
 }