private long InitializeLastDatabaseEtagOnIndexCreation(TransactionOperationContext indexContext) { const string key = "LastEtag"; if (_environment.IsNew == false) { var tree = indexContext.Transaction.InnerTransaction.ReadTree(IndexSchema.LastDocumentEtagOnIndexCreationTree); var result = tree?.Read(key); return(result?.Reader.ReadLittleEndianInt64() ?? 0); } using (var queryContext = QueryOperationContext.Allocate(DocumentDatabase, _index)) using (queryContext.OpenReadTransaction()) using (Slice.From(indexContext.Allocator, key, out var slice)) { var lastDatabaseEtag = DocumentsStorage.ReadLastEtag(queryContext.Documents.Transaction.InnerTransaction); var tree = indexContext.Transaction.InnerTransaction.CreateTree(IndexSchema.LastDocumentEtagOnIndexCreationTree); tree.Add(slice, lastDatabaseEtag); return(lastDatabaseEtag); } }
public static bool ShouldReplace(Index index, ref bool?isSideBySide) { if (isSideBySide.HasValue == false) { isSideBySide = index.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase); } if (isSideBySide == false) { return(false); } using (var context = QueryOperationContext.Allocate(index.DocumentDatabase, index)) using (index._contextPool.AllocateOperationContext(out TransactionOperationContext indexContext)) { using (indexContext.OpenReadTransaction()) using (context.OpenReadTransaction()) { indexContext.IgnoreStalenessDueToReduceOutputsToDelete = true; try { var canReplace = index.IsStale(context, indexContext) == false; if (canReplace) { isSideBySide = null; } return(canReplace); } finally { indexContext.IgnoreStalenessDueToReduceOutputsToDelete = false; } } } }
private static bool IsStaleDueToReferences(Index index, AbstractStaticIndexBase compiled, QueryOperationContext queryContext, TransactionOperationContext indexContext, long?referenceCutoff, long?compareExchangeReferenceCutoff, List <string> stalenessReasons) { foreach (var collection in index.Collections) { long lastIndexedEtag = -1; if (compiled.ReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections)) { lastIndexedEtag = index._indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); // we haven't handled references for that collection yet // in theory we could check what is the last etag for that collection in documents store // but this was checked earlier by the base index class if (lastIndexedEtag > 0) { foreach (var referencedCollection in referencedCollections) { var lastDocEtag = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(queryContext.Documents.Transaction.InnerTransaction, referencedCollection.Name); var lastProcessedReferenceEtag = index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); if (referenceCutoff == null) { if (lastDocEtag > lastProcessedReferenceEtag) { if (stalenessReasons == null) { return(true); } var lastDoc = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetByEtag(queryContext.Documents, lastDocEtag); stalenessReasons.Add($"There are still some document references to process from collection '{referencedCollection.Name}'. " + $"The last document etag in that collection is '{lastDocEtag:#,#;;0}' " + $"({Constants.Documents.Metadata.Id}: '{lastDoc.Id}', " + $"{Constants.Documents.Metadata.LastModified}: '{lastDoc.LastModified}'), " + $"but last processed document etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'."); } var lastTombstoneEtag = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetLastTombstoneEtag(queryContext.Documents.Transaction.InnerTransaction, referencedCollection.Name); if (lastTombstoneEtag > lastProcessedTombstoneEtag) { if (stalenessReasons == null) { return(true); } var lastTombstone = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetTombstoneByEtag(queryContext.Documents, lastTombstoneEtag); stalenessReasons.Add($"There are still some tombstone references to process from collection '{referencedCollection.Name}'. " + $"The last tombstone etag in that collection is '{lastTombstoneEtag:#,#;;0}' " + $"({Constants.Documents.Metadata.Id}: '{lastTombstone.LowerId}', " + $"{Constants.Documents.Metadata.LastModified}: '{lastTombstone.LastModified}'), " + $"but last processed tombstone etag for that collection is '{lastProcessedTombstoneEtag:#,#;;0}'."); } } else { var minDocEtag = Math.Min(referenceCutoff.Value, lastDocEtag); if (minDocEtag > lastProcessedReferenceEtag) { if (stalenessReasons == null) { return(true); } var lastDoc = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetByEtag(queryContext.Documents, lastDocEtag); stalenessReasons.Add($"There are still some document references to process from collection '{referencedCollection.Name}'. " + $"The last document etag in that collection is '{lastDocEtag:#,#;;0}' " + $"({Constants.Documents.Metadata.Id}: '{lastDoc.Id}', " + $"{Constants.Documents.Metadata.LastModified}: '{lastDoc.LastModified}') " + $"with cutoff set to '{referenceCutoff.Value}', " + $"but last processed document etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'."); } var hasTombstones = queryContext.Documents.DocumentDatabase.DocumentsStorage.HasTombstonesWithEtagGreaterThanStartAndLowerThanOrEqualToEnd(queryContext.Documents, referencedCollection.Name, lastProcessedTombstoneEtag, referenceCutoff.Value); if (hasTombstones) { if (stalenessReasons == null) { return(true); } stalenessReasons.Add($"There are still some tombstones to process from collection '{referencedCollection.Name}' with etag range '{lastProcessedTombstoneEtag} - {referenceCutoff.Value}'."); } } } } } if (compiled.CollectionsWithCompareExchangeReferences.Contains(collection)) { if (lastIndexedEtag == -1) { lastIndexedEtag = index._indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); } // we haven't handled references for that collection yet // in theory we could check what is the last etag for that collection in documents store // but this was checked earlier by the base index class if (lastIndexedEtag > 0) { var lastCompareExchangeEtag = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.GetLastCompareExchangeIndexForDatabase(queryContext.Server, queryContext.Documents.DocumentDatabase.Name); var lastProcessedReferenceEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange); if (compareExchangeReferenceCutoff == null) { if (lastCompareExchangeEtag > lastProcessedReferenceEtag) { if (stalenessReasons == null) { return(true); } stalenessReasons.Add($"There are still some compare exchange references to process for collection '{collection}'. The last compare exchange etag is '{lastCompareExchangeEtag:#,#;;0}', but last processed compare exchange etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'."); } var lastTombstoneEtag = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.GetLastCompareExchangeTombstoneIndexForDatabase(queryContext.Server, queryContext.Documents.DocumentDatabase.Name); var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange); if (lastTombstoneEtag > lastProcessedTombstoneEtag) { if (stalenessReasons == null) { return(true); } stalenessReasons.Add($"There are still some compare exchange tombstone references to process for collection '{collection}'. The last compare exchange tombstone etag is '{lastTombstoneEtag:#,#;;0}', but last processed compare exchange tombstone etag for that collection is '{lastProcessedTombstoneEtag:#,#;;0}'."); } } else { var minCompareExchangeEtag = Math.Min(compareExchangeReferenceCutoff.Value, lastCompareExchangeEtag); if (minCompareExchangeEtag > lastProcessedReferenceEtag) { if (stalenessReasons == null) { return(true); } stalenessReasons.Add($"There are still some compare exchange references to process for collection '{collection}'. The last compare exchange etag is '{lastCompareExchangeEtag:#,#;;0}' with cutoff set to '{compareExchangeReferenceCutoff.Value}', but last processed compare exchange etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'."); } var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange); var hasTombstones = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.HasCompareExchangeTombstonesWithEtagGreaterThanStartAndLowerThanOrEqualToEnd(queryContext.Server, queryContext.Documents.DocumentDatabase.Name, lastProcessedTombstoneEtag, compareExchangeReferenceCutoff.Value); if (hasTombstones) { if (stalenessReasons == null) { return(true); } stalenessReasons.Add($"There are still some compare exchange tombstones to process for collection '{collection}' with etag range '{lastProcessedTombstoneEtag} - {compareExchangeReferenceCutoff.Value}'."); } } } } } return(stalenessReasons?.Count > 0); }
public static bool IsStaleDueToReferences(MapReduceIndex index, QueryOperationContext queryContext, TransactionOperationContext indexContext, long?referenceCutoff, long?compareExchangeReferenceCutoff, List <string> stalenessReasons) { return(IsStaleDueToReferences(index, index._compiled, queryContext, indexContext, referenceCutoff, compareExchangeReferenceCutoff, stalenessReasons)); }
public static unsafe long CalculateIndexEtag(Index index, AbstractStaticIndexBase compiled, int length, byte *indexEtagBytes, byte *writePos, QueryOperationContext queryContext, TransactionOperationContext indexContext) { foreach (var collection in index.Collections) { if (compiled.ReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections)) { foreach (var referencedCollection in referencedCollections) { var lastDocEtag = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(queryContext.Documents.Transaction.InnerTransaction, referencedCollection.Name); var lastProcessedReferenceEtag = index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); var lastTombstoneEtag = queryContext.Documents.DocumentDatabase.DocumentsStorage.GetLastTombstoneEtag(queryContext.Documents.Transaction.InnerTransaction, referencedCollection.Name); var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForDocuments.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); *(long *)writePos = lastDocEtag; writePos += sizeof(long); *(long *)writePos = lastProcessedReferenceEtag; writePos += sizeof(long); *(long *)writePos = lastTombstoneEtag; writePos += sizeof(long); *(long *)writePos = lastProcessedTombstoneEtag; var referencesInfo = index.GetInMemoryReferencesState(collection, isCompareExchange: false); writePos += sizeof(long); *(long *)writePos = referencesInfo.ParentItemEtag; writePos += sizeof(long); *(long *)writePos = referencesInfo.ParentTombstoneEtag; } } if (compiled.CollectionsWithCompareExchangeReferences.Contains(collection)) { var lastCompareExchangeEtag = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.GetLastCompareExchangeIndexForDatabase(queryContext.Server, queryContext.Documents.DocumentDatabase.Name); var lastProcessedReferenceEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange); var lastTombstoneEtag = queryContext.Documents.DocumentDatabase.ServerStore.Cluster.GetLastCompareExchangeTombstoneIndexForDatabase(queryContext.Server, queryContext.Documents.DocumentDatabase.Name); var lastProcessedTombstoneEtag = index._indexStorage.ReferencesForCompareExchange.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection: IndexStorage.CompareExchangeReferences.CompareExchange); *(long *)writePos = lastCompareExchangeEtag; writePos += sizeof(long); *(long *)writePos = lastProcessedReferenceEtag; writePos += sizeof(long); *(long *)writePos = lastTombstoneEtag; writePos += sizeof(long); *(long *)writePos = lastProcessedTombstoneEtag; var referencesInfo = index.GetInMemoryReferencesState(collection, isCompareExchange: true); writePos += sizeof(long); *(long *)writePos = referencesInfo.ParentItemEtag; writePos += sizeof(long); *(long *)writePos = referencesInfo.ParentTombstoneEtag; } } unchecked { return((long)Hashing.XXHash64.Calculate(indexEtagBytes, (ulong)length)); } }
public static unsafe long CalculateIndexEtag(MapReduceIndex index, int length, byte *indexEtagBytes, byte *writePos, QueryOperationContext queryContext, TransactionOperationContext indexContext) { return(CalculateIndexEtag(index, index._compiled, length, indexEtagBytes, writePos, queryContext, indexContext)); }