private unsafe bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token)
{
var moreWorkFound = false;
Dictionary <string, long> lastIndexedEtagsByCollection = null;
foreach (var collection in _index.Collections)
{
if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false)
{
continue;
}
if (lastIndexedEtagsByCollection == null)
{
lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase);
}
if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false)
{
lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
}
if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now
{
continue;
}
foreach (var referencedCollection in referencedCollections)
{
using (var collectionStats = stats.For("Collection_" + referencedCollection.Name))
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Collection: {referencedCollection.Name}. Type: {actionType}.");
}
long lastReferenceEtag;
switch (actionType)
{
case ActionType.Document:
lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction, collection, referencedCollection);
break;
case ActionType.Tombstone:
lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection);
break;
default:
throw new NotSupportedException();
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. LastReferenceEtag: {lastReferenceEtag}.");
}
var lastEtag = lastReferenceEtag;
var count = 0;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var batchCount = 0;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
IEnumerable <Reference> references;
switch (actionType)
{
case ActionType.Document:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
.Select(document =>
{
_reference.Key = document.Id;
_reference.Etag = document.Etag;
return(_reference);
});
break;
case ActionType.Tombstone:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
.Select(tombstone =>
{
_reference.Key = tombstone.LowerId;
_reference.Etag = tombstone.Etag;
return(_reference);
});
break;
default:
throw new NotSupportedException();
}
foreach (var referencedDocument in references)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Processing reference: {referencedDocument.Key}.");
}
lastEtag = referencedDocument.Etag;
count++;
batchCount++;
var documents = new List <Document>();
foreach (var key in _indexStorage
.GetDocumentKeysFromCollectionThatReference(collection, referencedDocument.Key, indexContext.Transaction))
{
using (DocumentIdWorker.GetLower(databaseContext.Allocator, key.Content.Ptr, key.Size, out var loweredKey))
{
// when there is conflict, we need to apply same behavior as if the document would not exist
var doc = _documentsStorage.Get(databaseContext, loweredKey, throwOnConflict: false);
if (doc != null && doc.Etag <= lastIndexedEtag)
{
documents.Add(doc);
}
}
}
using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats))
{
while (docsEnumerator.MoveNext(out IEnumerable mapResults))
{
token.ThrowIfCancellationRequested();
var current = docsEnumerator.Current;
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Processing document: {current.Id}.");
}
try
{
_index.HandleMap(current.LowerId, mapResults, indexWriter, indexContext, collectionStats);
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
}
}
}
if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (batchCount == 0 || batchCount >= pageSize)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index} ({_index.Name})'. Processed {count} references in '{referencedCollection.Name}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
switch (actionType)
{
case ActionType.Document:
_indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
case ActionType.Tombstone:
_indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
default:
throw new NotSupportedException();
}
moreWorkFound = true;
}
}
}
return(moreWorkFound);
}
public virtual bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
const int pageSize = int.MaxValue;
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
var lastTombstoneEtag = _indexStorage.ReadLastProcessedTombstoneEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Collection: {collection}. LastMappedEtag: {lastMappedEtag:#,#;;0}. LastTombstoneEtag: {lastTombstoneEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastTombstoneEtag;
var count = 0;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var batchCount = 0;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
var tombstones = collection == Constants.Documents.Collections.AllDocumentsCollection
? _documentsStorage.GetTombstonesFrom(databaseContext, lastEtag + 1, 0, pageSize)
: _documentsStorage.GetTombstonesFrom(databaseContext, collection, lastEtag + 1, 0, pageSize);
foreach (var tombstone in tombstones)
{
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
count++;
batchCount++;
lastEtag = tombstone.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: true);
if (_logger.IsInfoEnabled && count % 2048 == 0)
{
_logger.Info($"Executing cleanup for '{_index.Name}'. Processed count: {count:#,#;;0} etag: {lastEtag}.");
}
if (tombstone.Type != Tombstone.TombstoneType.Document)
{
continue; // this can happen when we have '@all_docs'
}
_index.HandleDelete(tombstone, collection, indexWriter, indexContext, collectionStats);
if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (batchCount == 0 || batchCount >= pageSize)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processed {count} tombstones in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_indexStorage.WriteLastTombstoneEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedTombstoneEtags[collection] = lastEtag;
}
moreWorkFound = true;
}
}
return(moreWorkFound);
}
private bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token)
{
var moreWorkFound = false;
Dictionary <string, long> lastIndexedEtagsByCollection = null;
foreach (var collection in _index.Collections)
{
if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false)
{
continue;
}
if (lastIndexedEtagsByCollection == null)
{
lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase);
}
if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false)
{
lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
}
if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now
{
continue;
}
var totalProcessedCount = 0;
foreach (var referencedCollection in referencedCollections)
{
var inMemoryStats = _index.GetReferencesStats(referencedCollection.Name);
using (var collectionStats = stats.For("Collection_" + referencedCollection.Name))
{
long lastReferenceEtag;
switch (actionType)
{
case ActionType.Document:
lastReferenceEtag = IndexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
break;
case ActionType.Tombstone:
lastReferenceEtag = IndexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction.InnerTransaction, collection, referencedCollection);
break;
default:
throw new NotSupportedException();
}
var lastEtag = lastReferenceEtag;
var resultsCount = 0;
var sw = new Stopwatch();
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var hasChanges = false;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
IEnumerable <Reference> references;
switch (actionType)
{
case ActionType.Document:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize,
DocumentFields.Id | DocumentFields.Etag)
.Select(document =>
{
_reference.Key = document.Id;
_reference.Etag = document.Etag;
return(_reference);
});
break;
case ActionType.Tombstone:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
.Select(tombstone =>
{
_reference.Key = tombstone.LowerId;
_reference.Etag = tombstone.Etag;
return(_reference);
});
break;
default:
throw new NotSupportedException();
}
var isTombstone = actionType == ActionType.Tombstone;
foreach (var referencedDocument in references)
{
lastEtag = referencedDocument.Etag;
hasChanges = true;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone);
var documents = GetDocumentFromCollectionThatReference(databaseContext, indexContext, collection, referencedDocument, lastIndexedEtag);
using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats, _index.Type))
{
while (docsEnumerator.MoveNext(out IEnumerable mapResults))
{
token.ThrowIfCancellationRequested();
totalProcessedCount++;
collectionStats.RecordMapReferenceAttempt();
var current = docsEnumerator.Current;
stats.RecordDocumentSize(current.Data.Size);
try
{
var numberOfResults = _index.HandleMap(current.LowerId, current.Id, mapResults, writeOperation, indexContext, collectionStats);
resultsCount += numberOfResults;
collectionStats.RecordMapReferenceSuccess();
_index.MapsPerSec.MarkSingleThreaded(numberOfResults);
}
catch (Exception e) when(e.IsIndexError())
{
docsEnumerator.OnError();
_index.ErrorIndexIfCriticalException(e);
collectionStats.RecordMapReferenceError();
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
collectionStats.AddMapReferenceError(current.Id,
$"Failed to execute mapping function on {current.Id}. Exception: {e}");
}
_index.UpdateThreadAllocations(indexContext, writeOperation, stats, updateReduceStats: false);
}
}
if (CanContinueBatch(databaseContext, indexContext, collectionStats, writeOperation, lastEtag, lastCollectionEtag, totalProcessedCount) == false)
{
keepRunning = false;
break;
}
if (totalProcessedCount >= pageSize)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (hasChanges == false)
{
break;
}
}
}
if (lastReferenceEtag == lastEtag)
{
// the last referenced etag hasn't changed
continue;
}
moreWorkFound = true;
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executed handle references for '{_index.Name}' index and '{referencedCollection.Name}' collection. " +
$"Got {resultsCount:#,#;;0} map results in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
switch (actionType)
{
case ActionType.Document:
_indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
case ActionType.Tombstone:
_indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
default:
throw new NotSupportedException();
}
}
}
}
return(moreWorkFound);
}