public bool Execute(QueryOperationContext queryContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
var totalProcessedCount = 0;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name}'. Collection: {collection} LastMappedEtag: {lastMappedEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastMappedEtag;
var resultsCount = 0;
var pageSize = int.MaxValue;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
using (queryContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastItemEtagInCollection(queryContext, collection);
}
var items = GetItemsEnumerator(queryContext, collection, lastEtag, pageSize);
using (var itemEnumerator = _index.GetMapEnumerator(items, collection, indexContext, collectionStats, _index.Type))
{
while (true)
{
if (itemEnumerator.MoveNext(queryContext.Documents, out IEnumerable mapResults, out var etag) == false)
{
if (etag > lastEtag)
{
lastEtag = etag.Value;
}
collectionStats.RecordMapCompletedReason("No more documents to index");
keepRunning = false;
break;
}
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
var current = itemEnumerator.Current;
totalProcessedCount++;
collectionStats.RecordMapAttempt();
stats.RecordDocumentSize(current.Size);
if (_logger.IsInfoEnabled && totalProcessedCount % 8192 == 0)
{
_logger.Info($"Executing map for '{_index.Name}'. Processed count: {totalProcessedCount:#,#;;0} etag: {lastEtag:#,#;;0}.");
}
lastEtag = current.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: false);
try
{
var numberOfResults = _index.HandleMap(current, mapResults,
indexWriter, indexContext, collectionStats);
resultsCount += numberOfResults;
collectionStats.RecordMapSuccess();
_index.MapsPerSec?.MarkSingleThreaded(numberOfResults);
}
catch (Exception e) when(e.IsIndexError())
{
itemEnumerator.OnError();
_index.ErrorIndexIfCriticalException(e);
collectionStats.RecordMapError();
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
collectionStats.AddMapError(current.Id, $"Failed to execute mapping function on {current.Id}. " +
$"Exception: {e}");
}
if (CanContinueBatch(queryContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, totalProcessedCount) == false)
{
keepRunning = false;
break;
}
if (totalProcessedCount >= pageSize)
{
keepRunning = false;
break;
}
if (MaybeRenewTransaction(queryContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
}
}
}
if (lastMappedEtag == lastEtag)
{
// the last mapped etag hasn't changed
continue;
}
moreWorkFound = true;
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executed map for '{_index.Name}' index and '{collection}' collection. Got {resultsCount:#,#;;0} map results in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_index.SaveLastState();
_indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedDocEtags[collection] = lastEtag;
}
}
}
return(moreWorkFound);
}
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name} ({_index.IndexId})'. Collection: {collection}.");
}
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name} ({_index.IndexId})'. LastMappedEtag: {lastMappedEtag}.");
}
var lastEtag = lastMappedEtag;
var count = 0;
var resultsCount = 0;
var pageSize = int.MaxValue;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
}
var documents = GetDocumentsEnumerator(databaseContext, collection, lastEtag, pageSize);
using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats))
{
while (true)
{
IEnumerable mapResults;
if (docsEnumerator.MoveNext(out mapResults) == false)
{
collectionStats.RecordMapCompletedReason("No more documents to index");
keepRunning = false;
break;
}
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
var current = docsEnumerator.Current;
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Executing map for '{_index.Name} ({_index.IndexId})'. Processing document: {current.Key}.");
}
collectionStats.RecordMapAttempt();
count++;
lastEtag = current.Etag;
try
{
var numberOfResults = _index.HandleMap(current.LoweredKey, mapResults,
indexWriter, indexContext, collectionStats);
_index.MapsPerSec.Mark(numberOfResults);
resultsCount += numberOfResults;
collectionStats.RecordMapSuccess();
}
catch (Exception e)
{
_index.HandleError(e);
collectionStats.RecordMapError();
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Failed to execute mapping function on '{current.Key}' for '{_index.Name} ({_index.IndexId})'.",
e);
}
collectionStats.AddMapError(current.Key,
$"Failed to execute mapping function on {current.Key}. Exception: {e}");
}
if (CanContinueBatch(collectionStats, lastEtag, lastCollectionEtag) == false)
{
keepRunning = false;
break;
}
if (count >= pageSize)
{
keepRunning = false;
break;
}
if (MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name} ({_index.IndexId})'. Processed {count:#,#;;0} documents and {resultsCount:#,#;;0} map results in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedDocEtags[collection] = lastEtag;
}
moreWorkFound = true;
}
}
return(moreWorkFound);
}
public virtual bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
const int pageSize = int.MaxValue;
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
var lastTombstoneEtag = _indexStorage.ReadLastProcessedTombstoneEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Collection: {collection}. LastMappedEtag: {lastMappedEtag:#,#;;0}. LastTombstoneEtag: {lastTombstoneEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastTombstoneEtag;
var count = 0;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var batchCount = 0;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
var tombstones = collection == Constants.Documents.Collections.AllDocumentsCollection
? _documentsStorage.GetTombstonesFrom(databaseContext, lastEtag + 1, 0, pageSize)
: _documentsStorage.GetTombstonesFrom(databaseContext, collection, lastEtag + 1, 0, pageSize);
foreach (var tombstone in tombstones)
{
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
count++;
batchCount++;
lastEtag = tombstone.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: true);
if (_logger.IsInfoEnabled && count % 2048 == 0)
{
_logger.Info($"Executing cleanup for '{_index.Name}'. Processed count: {count:#,#;;0} etag: {lastEtag}.");
}
if (tombstone.Type != Tombstone.TombstoneType.Document)
{
continue; // this can happen when we have '@all_docs'
}
_index.HandleDelete(tombstone, collection, indexWriter, indexContext, collectionStats);
if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (batchCount == 0 || batchCount >= pageSize)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processed {count} tombstones in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_indexStorage.WriteLastTombstoneEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedTombstoneEtags[collection] = lastEtag;
}
moreWorkFound = true;
}
}
return(moreWorkFound);
}
public virtual (bool MoreWorkFound, Index.CanContinueBatchResult BatchContinuationResult) Execute(QueryOperationContext queryContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
const long pageSize = long.MaxValue;
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
var batchContinuationResult = Index.CanContinueBatchResult.None;
var totalProcessedCount = 0;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
var lastTombstoneEtag = _indexStorage.ReadLastProcessedTombstoneEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Collection: {collection}. LastMappedEtag: {lastMappedEtag:#,#;;0}. LastTombstoneEtag: {lastTombstoneEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastTombstoneEtag;
var count = 0;
var sw = new Stopwatch();
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var hasChanges = false;
using (queryContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(queryContext, collection);
}
var tombstones = collection == Constants.Documents.Collections.AllDocumentsCollection
? _documentsStorage.GetTombstonesFrom(queryContext.Documents, lastEtag + 1, 0, pageSize)
: _documentsStorage.GetTombstonesFrom(queryContext.Documents, collection, lastEtag + 1, 0, pageSize);
foreach (var tombstone in tombstones)
{
token.ThrowIfCancellationRequested();
count++;
totalProcessedCount++;
hasChanges = true;
lastEtag = tombstone.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: true);
if (_logger.IsInfoEnabled && totalProcessedCount % 2048 == 0)
{
_logger.Info($"Executing cleanup for '{_index.Name}'. Processed count: {totalProcessedCount:#,#;;0} etag: {lastEtag}.");
}
if (tombstone.Type != Tombstone.TombstoneType.Document)
{
continue; // this can happen when we have '@all_docs'
}
_index.HandleDelete(tombstone, collection, writeOperation, indexContext, collectionStats);
stats.RecordTombstoneDeleteSuccess();
var parameters = new CanContinueBatchParameters(stats, IndexingWorkType.Cleanup, queryContext, indexContext, writeOperation, lastEtag,
lastCollectionEtag,
totalProcessedCount, sw);
batchContinuationResult = _index.CanContinueBatch(in parameters, ref maxTimeForDocumentTransactionToRemainOpen);
if (batchContinuationResult != Index.CanContinueBatchResult.True)
{
keepRunning = batchContinuationResult == Index.CanContinueBatchResult.RenewTransaction;
break;
}
}
if (hasChanges == false)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processed {count} tombstones in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_indexStorage.WriteLastTombstoneEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedTombstoneEtags[collection] = lastEtag;
}
moreWorkFound = true;
}
}
return(moreWorkFound, batchContinuationResult);
}