public static Dictionary <string, long> GetLastProcessedTombstonesPerCollection(
Index index, HashSet <string> referencedCollections, HashSet <string> collections,
Dictionary <string, HashSet <CollectionName> > compiledReferencedCollections,
IndexStorage indexStorage)
{
using (index._contextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenReadTransaction())
{
var etags = index.GetLastProcessedDocumentTombstonesPerCollection(tx);
if (referencedCollections.Count <= 0)
{
return(etags);
}
foreach (var collection in collections)
{
if (compiledReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> collectionNames) == false)
{
continue;
}
foreach (var collectionName in collectionNames)
{
var etag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(tx.InnerTransaction, collection, collectionName);
if (etags.TryGetValue(collectionName.Name, out long currentEtag) == false || etag < currentEtag)
{
etags[collectionName.Name] = etag;
}
}
}
return(etags);
}
}
private static unsafe long CalculateIndexEtag(Index index, StaticIndexBase compiled, int length, byte *indexEtagBytes, byte *writePos, DocumentsOperationContext documentsContext, TransactionOperationContext indexContext)
{
foreach (var collection in index.Collections)
{
if (compiled.ReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false)
{
continue;
}
foreach (var referencedCollection in referencedCollections)
{
var lastDocEtag = documentsContext.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(documentsContext.Transaction.InnerTransaction, referencedCollection.Name);
var lastProcessedReferenceEtag = IndexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
var lastTombstoneEtag = documentsContext.DocumentDatabase.DocumentsStorage.GetLastTombstoneEtag(documentsContext.Transaction.InnerTransaction, referencedCollection.Name);
var lastProcessedTombstoneEtag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
*(long *)writePos = lastDocEtag;
writePos += sizeof(long);
*(long *)writePos = lastProcessedReferenceEtag;
writePos += sizeof(long);
*(long *)writePos = lastTombstoneEtag;
writePos += sizeof(long);
*(long *)writePos = lastProcessedTombstoneEtag;
}
}
unchecked
{
return((long)Hashing.XXHash64.Calculate(indexEtagBytes, (ulong)length));
}
}
private static bool IsStaleDueToReferences(Index index, StaticIndexBase compiled, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, long?referenceCutoff, List <string> stalenessReasons)
{
foreach (var collection in index.Collections)
{
if (compiled.ReferencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false)
{
continue;
}
var lastIndexedEtag = index._indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
// we haven't handled references for that collection yet
// in theory we could check what is the last etag for that collection in documents store
// but this was checked earlier by the base index class
if (lastIndexedEtag == 0)
{
continue;
}
foreach (var referencedCollection in referencedCollections)
{
var lastDocEtag = databaseContext.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(databaseContext.Transaction.InnerTransaction, referencedCollection.Name);
var lastProcessedReferenceEtag = IndexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
var lastProcessedTombstoneEtag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
if (referenceCutoff == null)
{
if (lastDocEtag > lastProcessedReferenceEtag)
{
if (stalenessReasons == null)
{
return(true);
}
var lastDoc = databaseContext.DocumentDatabase.DocumentsStorage.GetByEtag(databaseContext, lastDocEtag);
stalenessReasons.Add($"There are still some document references to process from collection '{referencedCollection.Name}'. " +
$"The last document etag in that collection is '{lastDocEtag:#,#;;0}' " +
$"({Constants.Documents.Metadata.Id}: '{lastDoc.Id}', " +
$"{Constants.Documents.Metadata.LastModified}: '{lastDoc.LastModified}'), " +
$"but last processed document etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'.");
}
var lastTombstoneEtag = databaseContext.DocumentDatabase.DocumentsStorage.GetLastTombstoneEtag(databaseContext.Transaction.InnerTransaction, referencedCollection.Name);
if (lastTombstoneEtag > lastProcessedTombstoneEtag)
{
if (stalenessReasons == null)
{
return(true);
}
var lastTombstone = databaseContext.DocumentDatabase.DocumentsStorage.GetTombstoneByEtag(databaseContext, lastTombstoneEtag);
stalenessReasons.Add($"There are still some tombstone references to process from collection '{referencedCollection.Name}'. " +
$"The last tombstone etag in that collection is '{lastTombstoneEtag:#,#;;0}' " +
$"({Constants.Documents.Metadata.Id}: '{lastTombstone.LowerId}', " +
$"{Constants.Documents.Metadata.LastModified}: '{lastTombstone.LastModified}'), " +
$"but last processed tombstone etag for that collection is '{lastProcessedTombstoneEtag:#,#;;0}'.");
}
}
else
{
var minDocEtag = Math.Min(referenceCutoff.Value, lastDocEtag);
if (minDocEtag > lastProcessedReferenceEtag)
{
if (stalenessReasons == null)
{
return(true);
}
var lastDoc = databaseContext.DocumentDatabase.DocumentsStorage.GetByEtag(databaseContext, lastDocEtag);
stalenessReasons.Add($"There are still some document references to process from collection '{referencedCollection.Name}'. " +
$"The last document etag in that collection is '{lastDocEtag:#,#;;0}' " +
$"({Constants.Documents.Metadata.Id}: '{lastDoc.Id}', " +
$"{Constants.Documents.Metadata.LastModified}: '{lastDoc.LastModified}') " +
$"with cutoff set to '{referenceCutoff.Value}', " +
$"but last processed document etag for that collection is '{lastProcessedReferenceEtag:#,#;;0}'.");
}
var hasTombstones = databaseContext.DocumentDatabase.DocumentsStorage.HasTombstonesWithEtagGreaterThanStartAndLowerThanOrEqualToEnd(databaseContext, referencedCollection.Name,
lastProcessedTombstoneEtag,
referenceCutoff.Value);
if (hasTombstones)
{
if (stalenessReasons == null)
{
return(true);
}
stalenessReasons.Add($"There are still tombstones to process from collection '{referencedCollection.Name}' with etag range '{lastProcessedTombstoneEtag} - {referenceCutoff.Value}'.");
}
}
}
}
return(stalenessReasons?.Count > 0);
}