public static Dictionary <string, long> GetLastProcessedTombstonesPerCollection( Index index, HashSet <string> referencedCollections, HashSet <string> collections, Dictionary <string, HashSet <CollectionName> > compiledReferencedCollections, IndexStorage indexStorage) { using (index._contextPool.AllocateOperationContext(out TransactionOperationContext context)) using (var tx = context.OpenReadTransaction()) { var etags = index.GetLastProcessedDocumentTombstonesPerCollection(tx); if (referencedCollections.Count <= 0) { return(etags); } foreach (var collection in collections) { if (compiledReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> collectionNames) == false) { continue; } foreach (var collectionName in collectionNames) { var etag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(tx.InnerTransaction, collection, collectionName); if (etags.TryGetValue(collectionName.Name, out long currentEtag) == false || etag < currentEtag) { etags[collectionName.Name] = etag; } } } return(etags); } }
private static unsafe long CalculateIndexEtag(Index index, StaticIndexBase compiled, int length, byte *indexEtagBytes, byte *writePos, DocumentsOperationContext documentsContext, TransactionOperationContext indexContext) { foreach (var collection in index.Collections) { if (compiled.ReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false) { continue; } foreach (var referencedCollection in referencedCollections) { var lastDocEtag = documentsContext.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(documentsContext.Transaction.InnerTransaction, referencedCollection.Name); var lastProcessedReferenceEtag = IndexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); var lastTombstoneEtag = documentsContext.DocumentDatabase.DocumentsStorage.GetLastTombstoneEtag(documentsContext.Transaction.InnerTransaction, referencedCollection.Name); var lastProcessedTombstoneEtag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); *(long *)writePos = lastDocEtag; writePos += sizeof(long); *(long *)writePos = lastProcessedReferenceEtag; writePos += sizeof(long); *(long *)writePos = lastTombstoneEtag; writePos += sizeof(long); *(long *)writePos = lastProcessedTombstoneEtag; } } unchecked { return((long)Hashing.XXHash64.Calculate(indexEtagBytes, (ulong)length)); } }
private static bool IsStaleDueToReferences(Index index, StaticIndexBase compiled, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, long?referenceCutoff, List <string> stalenessReasons) { foreach (var collection in index.Collections) { if (compiled.ReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false) { continue; } var lastIndexedEtag = index._indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection); // we haven't handled references for that collection yet // in theory we could check what is the last etag for that collection in documents store // but this was checked earlier by the base index class if (lastIndexedEtag == 0) { continue; } foreach (var referencedCollection in referencedCollections) { var lastDocEtag = databaseContext.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(databaseContext.Transaction.InnerTransaction, referencedCollection.Name); var lastProcessedReferenceEtag = IndexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); var lastProcessedTombstoneEtag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection); if (referenceCutoff == null) { if (lastDocEtag > lastProcessedReferenceEtag) { if (stalenessReasons == null) { return(true); } var lastDoc = databaseContext.DocumentDatabase.DocumentsStorage.GetByEtag(databaseContext, lastDocEtag); stalenessReasons.Add($"There are still some document references to process from collection '{referencedCollection.Name}'. " + $"The last document etag in that collection is '{lastDocEtag:#,#;;0}' " + $"({Constants.Documents.Metadata.Id}: '{lastDoc.Id}', " + $"{Constants.Documents.Metadata.LastModified}: '{lastDoc.LastModified}'), " + $"but last processed document etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'."); } var lastTombstoneEtag = databaseContext.DocumentDatabase.DocumentsStorage.GetLastTombstoneEtag(databaseContext.Transaction.InnerTransaction, referencedCollection.Name); if (lastTombstoneEtag > lastProcessedTombstoneEtag) { if (stalenessReasons == null) { return(true); } var lastTombstone = databaseContext.DocumentDatabase.DocumentsStorage.GetTombstoneByEtag(databaseContext, lastTombstoneEtag); stalenessReasons.Add($"There are still some tombstone references to process from collection '{referencedCollection.Name}'. " + $"The last tombstone etag in that collection is '{lastTombstoneEtag:#,#;;0}' " + $"({Constants.Documents.Metadata.Id}: '{lastTombstone.LowerId}', " + $"{Constants.Documents.Metadata.LastModified}: '{lastTombstone.LastModified}'), " + $"but last processed tombstone etag for that collection is '{lastProcessedTombstoneEtag:#,#;;0}'."); } } else { var minDocEtag = Math.Min(referenceCutoff.Value, lastDocEtag); if (minDocEtag > lastProcessedReferenceEtag) { if (stalenessReasons == null) { return(true); } var lastDoc = databaseContext.DocumentDatabase.DocumentsStorage.GetByEtag(databaseContext, lastDocEtag); stalenessReasons.Add($"There are still some document references to process from collection '{referencedCollection.Name}'. " + $"The last document etag in that collection is '{lastDocEtag:#,#;;0}' " + $"({Constants.Documents.Metadata.Id}: '{lastDoc.Id}', " + $"{Constants.Documents.Metadata.LastModified}: '{lastDoc.LastModified}') " + $"with cutoff set to '{referenceCutoff.Value}', " + $"but last processed document etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'."); } var hasTombstones = databaseContext.DocumentDatabase.DocumentsStorage.HasTombstonesWithEtagGreaterThanStartAndLowerThanOrEqualToEnd(databaseContext, referencedCollection.Name, lastProcessedTombstoneEtag, referenceCutoff.Value); if (hasTombstones) { if (stalenessReasons == null) { return(true); } stalenessReasons.Add($"There are still tombstones to process from collection '{referencedCollection.Name}' with etag range '{lastProcessedTombstoneEtag} - {referenceCutoff.Value}'."); } } } } return(stalenessReasons?.Count > 0); }