public override int HandleMap(IndexItem indexItem, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
EnsureValidStats(stats);
bool mustDelete;
using (_stats.BloomStats.Start())
{
mustDelete = _filters.Add(indexItem.LowerId) == false;
}
if (indexItem.SkipLuceneDelete == false && mustDelete)
{
writer.Delete(indexItem.LowerId, stats);
}
var numberOfOutputs = 0;
foreach (var mapResult in mapResults)
{
writer.IndexDocument(indexItem.LowerId, indexItem.LowerSourceDocumentId, mapResult, stats, indexContext);
numberOfOutputs++;
}
HandleIndexOutputsPerDocument(indexItem.Id ?? indexItem.LowerId, numberOfOutputs, stats);
DocumentDatabase.Metrics.MapIndexes.IndexedPerSec.Mark(numberOfOutputs);
return(numberOfOutputs);
}
public override int HandleMap(LazyStringValue key, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
EnsureValidStats(stats);
bool mustDelete;
using (_stats.BloomStats.Start())
{
mustDelete = _filter.Add(key) == false;
}
if (mustDelete)
{
writer.Delete(key, stats);
}
var numberOfOutputs = 0;
foreach (var mapResult in mapResults)
{
writer.IndexDocument(key, mapResult, stats, indexContext);
numberOfOutputs++;
}
HandleIndexOutputsPerDocument(key, numberOfOutputs, stats);
DocumentDatabase.Metrics.IndexedPerSecond.Mark();
return(numberOfOutputs);
}
public bool CanContinueBatch(
DocumentsOperationContext documentsContext,
TransactionOperationContext indexingContext,
IndexingStatsScope stats,
IndexWriteOperation indexWriteOperation,
long currentEtag,
long maxEtag,
int count)
{
if (stats.Duration >= _configuration.MapTimeout.AsTimeSpan)
{
return(false);
}
if (currentEtag >= maxEtag && stats.Duration >= _configuration.MapTimeoutAfterEtagReached.AsTimeSpan)
{
return(false);
}
if (_index.CanContinueBatch(stats, documentsContext, indexingContext, indexWriteOperation, count) == false)
{
return(false);
}
return(true);
}
public bool CanContinueBatch(DocumentsOperationContext documentsContext, TransactionOperationContext indexingContext,
IndexingStatsScope stats, IndexWriteOperation indexWriteOperation, long currentEtag, long maxEtag, int count)
{
if (stats.Duration >= _configuration.MapTimeout.AsTimeSpan)
{
return(false);
}
if (_configuration.MapBatchSize.HasValue && count >= _configuration.MapBatchSize.Value)
{
return(false);
}
if (currentEtag >= maxEtag && stats.Duration >= _configuration.MapTimeoutAfterEtagReached.AsTimeSpan)
{
return(false);
}
if (_index.ShouldReleaseTransactionBecauseFlushIsWaiting(stats))
{
return(false);
}
if (_index.CanContinueBatch(stats, documentsContext, indexingContext, indexWriteOperation, count) == false)
{
return(false);
}
return(true);
}
private void HandleNestedValuesReduction(TransactionOperationContext indexContext, IndexingStatsScope stats,
CancellationToken token, MapReduceResultsStore modifiedStore,
IndexWriteOperation writer, LazyStringValue reduceKeyHash)
{
EnsureValidNestedValuesReductionStats(stats);
var numberOfEntriesToReduce = 0;
try
{
var section = modifiedStore.GetNestedResultsSection();
if (section.IsModified == false)
{
return;
}
using (_nestedValuesReductionStats.NestedValuesRead.Start())
{
numberOfEntriesToReduce += section.GetResults(indexContext, _aggregationBatch.Items);
}
stats.RecordReduceAttempts(numberOfEntriesToReduce);
AggregationResult result;
using (_nestedValuesReductionStats.NestedValuesAggregation.Start())
{
result = AggregateOn(_aggregationBatch.Items, indexContext, token);
}
if (section.IsNew == false)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
}
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
_index.ReducesPerSec.Mark(numberOfEntriesToReduce);
_metrics.MapReduceIndexes.ReducedPerSec.Mark(numberOfEntriesToReduce);
stats.RecordReduceSuccesses(numberOfEntriesToReduce);
}
catch (Exception e)
{
_index.HandleError(e);
var message = $"Failed to execute reduce function for reduce key '{reduceKeyHash}' on nested values of '{_indexDefinition.Name}' index.";
if (_logger.IsInfoEnabled)
{
_logger.Info(message, e);
}
stats.RecordReduceErrors(numberOfEntriesToReduce);
stats.AddReduceError(message + $" Exception: {e}");
}
}
public override void HandleDelete(Tombstone tombstone, string collection, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
if (_referencedCollections.Count > 0)
_handleReferences.HandleDelete(tombstone, collection, writer, indexContext, stats);
base.HandleDelete(tombstone, collection, writer, indexContext, stats);
}
public unsafe void HandleDelete(Tombstone tombstone, string collection, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
var tx = indexContext.Transaction.InnerTransaction;
var loweredKey = tombstone.LowerId;
using (Slice.External(tx.Allocator, loweredKey, out Slice tombstoneKeySlice))
_indexStorage.RemoveReferences(tombstoneKeySlice, collection, null, indexContext.Transaction);
}
public override int HandleMap(LazyStringValue key, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
EnsureValidStats(stats);
var mappedResult = new DynamicJsonValue();
using (_stats.BlittableJsonAggregation.Start())
{
var document = ((Document[])mapResults)[0];
Debug.Assert(key == document.LowerId);
foreach (var field in Definition.MapFields.Values)
{
var autoIndexField = field.As <AutoIndexField>();
switch (autoIndexField.Aggregation)
{
case AggregationOperation.Count:
mappedResult[autoIndexField.Name] = 1;
break;
case AggregationOperation.Sum:
object fieldValue;
BlittableJsonTraverser.Default.TryRead(document.Data, autoIndexField.Name, out fieldValue, out _);
var arrayResult = fieldValue as IEnumerable <object>;
if (arrayResult == null)
{
// explicitly adding this even if the value isn't there, as a null
mappedResult[autoIndexField.Name] = fieldValue;
continue;
}
decimal total = 0;
foreach (var item in arrayResult)
{
if (item == null)
{
continue;
}
switch (BlittableNumber.Parse(item, out double doubleValue, out long longValue))
{
case NumberParseResult.Double:
total += (decimal)doubleValue;
break;
case NumberParseResult.Long:
total += longValue;
break;
}
}
mappedResult[autoIndexField.Name] = total;
break;
public override int HandleMap(IndexItem indexItem, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
if (_enumerationWrappers.TryGetValue(CurrentIndexingScope.Current.SourceCollection, out AnonymousObjectToBlittableMapResultsEnumerableWrapper wrapper) == false)
{
_enumerationWrappers[CurrentIndexingScope.Current.SourceCollection] = wrapper = new AnonymousObjectToBlittableMapResultsEnumerableWrapper(this, indexContext);
}
wrapper.InitializeForEnumeration(mapResults, indexContext, stats);
return(PutMapResults(indexItem.LowerId, indexItem.SourceDocumentId, wrapper, indexContext, stats));
}
private void HandleNestedValuesReduction(TransactionOperationContext indexContext, IndexingStatsScope stats,
MapReduceResultsStore modifiedStore,
IndexWriteOperation writer, LazyStringValue reduceKeyHash, CancellationToken token)
{
EnsureValidNestedValuesReductionStats(stats);
var numberOfEntriesToReduce = 0;
try
{
var section = modifiedStore.GetNestedResultsSection();
if (section.IsModified == false)
{
return;
}
using (_nestedValuesReductionStats.NestedValuesRead.Start())
{
numberOfEntriesToReduce += section.GetResults(indexContext, _aggregationBatch.Items);
}
stats.RecordReduceAttempts(numberOfEntriesToReduce);
AggregationResult result;
using (_nestedValuesReductionStats.NestedValuesAggregation.Start())
{
result = AggregateOn(_aggregationBatch.Items, indexContext, _nestedValuesReductionStats.NestedValuesAggregation, token);
}
if (section.IsNew == false)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
}
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
_index.ReducesPerSec.MarkSingleThreaded(numberOfEntriesToReduce);
_metrics.MapReduceIndexes.ReducedPerSec.Mark(numberOfEntriesToReduce);
stats.RecordReduceSuccesses(numberOfEntriesToReduce);
_index.UpdateThreadAllocations(indexContext, writer, stats, updateReduceStats: true);
}
catch (Exception e) when(e.IsIndexError())
{
_index.ErrorIndexIfCriticalException(e);
HandleReductionError(e, reduceKeyHash, writer, stats, updateStats: true, page: null, numberOfNestedValues: numberOfEntriesToReduce);
}
}
public override int HandleMap(IndexItem indexItem, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
EnsureValidStats(stats);
var document = ((Document[])mapResults)[0];
Debug.Assert(indexItem.LowerId == document.LowerId);
using (_stats.BlittableJsonAggregation.Start())
{
DynamicJsonValue singleResult = null;
var groupByFieldsCount = Definition.OrderedGroupByFields.Length;
for (var i = 0; i < groupByFieldsCount; i++)
{
var groupByField = Definition.OrderedGroupByFields[i];
BlittableJsonTraverserHelper.TryRead(BlittableJsonTraverser.Default, document, groupByField.Name, out object result);
if (_isFanout == false)
{
if (singleResult == null)
{
singleResult = new DynamicJsonValue();
}
singleResult[groupByField.Name] = result;
_reduceKeyProcessor.Process(indexContext.Allocator, result);
}
else
{
if (result is IEnumerable array && groupByField.GroupByArrayBehavior == GroupByArrayBehavior.ByIndividualValues)
{
// fanout
foreach (var item in array)
{
_output.AddGroupByValue(groupByField.Name, item);
}
}
public override int HandleMap(LazyStringValue key, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
EnsureValidStats(stats);
bool mustDelete;
using (_stats.BloomStats.Start())
{
mustDelete = _filter.Add(key) == false;
}
if (mustDelete)
{
writer.Delete(key, stats);
}
var numberOfOutputs = 0;
foreach (var mapResult in mapResults)
{
writer.IndexDocument(key, mapResult, stats, indexContext);
numberOfOutputs++;
if (EnsureValidNumberOfOutputsForDocument(numberOfOutputs))
{
continue;
}
writer.Delete(key, stats);
throw new InvalidOperationException($"Index '{Name}' has already produced {numberOfOutputs} map results for a source document '{key}', while the allowed max number of outputs is {MaxNumberOfIndexOutputs} per one document. Please verify this index definition and consider a re-design of your entities or index.");
}
DocumentDatabase.Metrics.IndexedPerSecond.Mark();
return(numberOfOutputs);
}
public bool CanContinueBatch(QueryOperationContext queryContext, TransactionOperationContext indexingContext, IndexingStatsScope stats, IndexWriteOperation indexWriter, long currentEtag, long maxEtag, long count)
{
if (stats.Duration >= _configuration.MapTimeout.AsTimeSpan)
{
stats.RecordMapCompletedReason($"Exceeded maximum configured map duration ({_configuration.MapTimeout.AsTimeSpan}). Was {stats.Duration}");
return(false);
}
if (_configuration.MapBatchSize.HasValue && count >= _configuration.MapBatchSize.Value)
{
stats.RecordMapCompletedReason($"Reached maximum configured map batch size ({_configuration.MapBatchSize.Value:#,#;;0}).");
return(false);
}
if (currentEtag >= maxEtag && stats.Duration >= _configuration.MapTimeoutAfterEtagReached.AsTimeSpan)
{
stats.RecordMapCompletedReason($"Reached maximum etag that was seen when batch started ({maxEtag:#,#;;0}) and map duration ({stats.Duration}) exceeded configured limit ({_configuration.MapTimeoutAfterEtagReached.AsTimeSpan})");
return(false);
}
if (_index.ShouldReleaseTransactionBecauseFlushIsWaiting(stats))
{
return(false);
}
if (_index.CanContinueBatch(stats, queryContext, indexingContext, indexWriter, count) == false)
{
return(false);
}
return(true);
}
public virtual bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
const int pageSize = int.MaxValue;
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
var lastTombstoneEtag = _indexStorage.ReadLastProcessedTombstoneEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Collection: {collection}. LastMappedEtag: {lastMappedEtag:#,#;;0}. LastTombstoneEtag: {lastTombstoneEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastTombstoneEtag;
var count = 0;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var batchCount = 0;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
var tombstones = collection == Constants.Documents.Collections.AllDocumentsCollection
? _documentsStorage.GetTombstonesFrom(databaseContext, lastEtag + 1, 0, pageSize)
: _documentsStorage.GetTombstonesFrom(databaseContext, collection, lastEtag + 1, 0, pageSize);
foreach (var tombstone in tombstones)
{
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
count++;
batchCount++;
lastEtag = tombstone.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: true);
if (_logger.IsInfoEnabled && count % 2048 == 0)
{
_logger.Info($"Executing cleanup for '{_index.Name}'. Processed count: {count:#,#;;0} etag: {lastEtag}.");
}
if (tombstone.Type != Tombstone.TombstoneType.Document)
{
continue; // this can happen when we have '@all_docs'
}
_index.HandleDelete(tombstone, collection, indexWriter, indexContext, collectionStats);
if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (batchCount == 0 || batchCount >= pageSize)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processed {count} tombstones in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_indexStorage.WriteLastTombstoneEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedTombstoneEtags[collection] = lastEtag;
}
moreWorkFound = true;
}
}
return(moreWorkFound);
}
public override unsafe void HandleDelete(Tombstone tombstone, string collection, IndexWriteOperation writer,
TransactionOperationContext indexContext, IndexingStatsScope stats)
{
using (Slice.External(indexContext.Allocator, tombstone.LowerId.Buffer, tombstone.LowerId.Length, out Slice docKeyAsSlice))
{
MapReduceWorkContext.DocumentMapEntries.RepurposeInstance(docKeyAsSlice, clone: false);
if (MapReduceWorkContext.DocumentMapEntries.NumberOfEntries == 0)
{
return;
}
foreach (var mapEntry in GetMapEntries(MapReduceWorkContext.DocumentMapEntries))
{
var store = GetResultsStore(mapEntry.ReduceKeyHash, indexContext, create: false);
store.Delete(mapEntry.Id);
}
MapReduceWorkContext.MapPhaseTree.DeleteFixedTreeFor(tombstone.LowerId, sizeof(ulong));
}
}
public override int HandleMap(LazyStringValue lowerId, LazyStringValue id, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
throw new NotImplementedException();
}
public override void HandleDelete(Tombstone tombstone, string collection, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
writer.Delete(tombstone.LowerId, stats);
}
public override void HandleDelete(DocumentTombstone tombstone, string collection, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e);
}
private void HandleReductionError(Exception error, LazyStringValue reduceKeyHash, IndexWriteOperation writer, IndexingStatsScope stats, bool updateStats, TreePage page,
int numberOfNestedValues = -1)
{
var builder = new StringBuilder("Failed to execute reduce function on ");
if (page != null)
{
builder.Append($"page {page} ");
}
else
{
builder.Append("nested values ");
}
builder.Append($"of '{_indexDefinition.Name}' index. The relevant reduce result is going to be removed from the index ");
builder.Append($"as it would be incorrect due to encountered errors (reduce key hash: {reduceKeyHash}");
var sampleItem = _aggregationBatch?.Items?.FirstOrDefault();
if (sampleItem != null)
{
builder.Append($", sample item to reduce: {sampleItem}");
}
builder.Append(")");
var message = builder.ToString();
if (_logger.IsInfoEnabled)
{
_logger.Info(message, error);
}
try
{
writer.DeleteReduceResult(reduceKeyHash, stats);
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to delete an index result from '${_indexDefinition.Name}' index on reduce error (reduce key hash: ${reduceKeyHash})", e);
}
}
if (updateStats)
{
var numberOfEntries = page?.NumberOfEntries ?? numberOfNestedValues;
Debug.Assert(numberOfEntries != -1);
// we'll only want to record exceptions on some of these, to give the
// user information about what is going on, otherwise we'll have to wait a lot if we
// are processing a big batch, and this can be a perf killer. See: RavenDB-11038
stats.RecordReduceErrors(numberOfEntries);
if (stats.NumberOfKeptReduceErrors < IndexStorage.MaxNumberOfKeptErrors)
{
stats.AddReduceError(message + $" Exception: {error}");
}
var failureInfo = new IndexFailureInformation
{
Name = _index.Name,
MapErrors = stats.MapErrors,
MapAttempts = stats.MapAttempts,
ReduceErrors = stats.ReduceErrors,
ReduceAttempts = stats.ReduceAttempts
};
if (failureInfo.IsInvalidIndex(_isStaleBecauseOfRunningReduction))
{
throw new ExcessiveNumberOfReduceErrorsException("Excessive number of errors during the reduce phase for the current batch. Failure info: " +
failureInfo.GetErrorMessage());
}
}
}
public override int HandleMap(IndexItem indexItem, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
throw new System.NotImplementedException();
}
private unsafe bool HandleDocuments(ActionType actionType, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, int pageSize, TimeSpan maxTimeForDocumentTransactionToRemainOpen, CancellationToken token)
{
var moreWorkFound = false;
Dictionary <string, long> lastIndexedEtagsByCollection = null;
foreach (var collection in _index.Collections)
{
if (_referencedCollections.TryGetValue(collection, out HashSet <CollectionName> referencedCollections) == false)
{
continue;
}
if (lastIndexedEtagsByCollection == null)
{
lastIndexedEtagsByCollection = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase);
}
if (lastIndexedEtagsByCollection.TryGetValue(collection, out long lastIndexedEtag) == false)
{
lastIndexedEtagsByCollection[collection] = lastIndexedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
}
if (lastIndexedEtag == 0) // we haven't indexed yet, so we are skipping references for now
{
continue;
}
foreach (var referencedCollection in referencedCollections)
{
using (var collectionStats = stats.For("Collection_" + referencedCollection.Name))
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Collection: {referencedCollection.Name}. Type: {actionType}.");
}
long lastReferenceEtag;
switch (actionType)
{
case ActionType.Document:
lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceEtag(indexContext.Transaction, collection, referencedCollection);
break;
case ActionType.Tombstone:
lastReferenceEtag = _indexStorage.ReadLastProcessedReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection);
break;
default:
throw new NotSupportedException();
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. LastReferenceEtag: {lastReferenceEtag}.");
}
var lastEtag = lastReferenceEtag;
var count = 0;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var batchCount = 0;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
IEnumerable <Reference> references;
switch (actionType)
{
case ActionType.Document:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetDocumentsFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
.Select(document =>
{
_reference.Key = document.Id;
_reference.Etag = document.Etag;
return(_reference);
});
break;
case ActionType.Tombstone:
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
references = _documentsStorage
.GetTombstonesFrom(databaseContext, referencedCollection.Name, lastEtag + 1, 0, pageSize)
.Select(tombstone =>
{
_reference.Key = tombstone.LowerId;
_reference.Etag = tombstone.Etag;
return(_reference);
});
break;
default:
throw new NotSupportedException();
}
foreach (var referencedDocument in references)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Processing reference: {referencedDocument.Key}.");
}
lastEtag = referencedDocument.Etag;
count++;
batchCount++;
var documents = new List <Document>();
foreach (var key in _indexStorage
.GetDocumentKeysFromCollectionThatReference(collection, referencedDocument.Key, indexContext.Transaction))
{
using (DocumentIdWorker.GetLower(databaseContext.Allocator, key.Content.Ptr, key.Size, out var loweredKey))
{
// when there is conflict, we need to apply same behavior as if the document would not exist
var doc = _documentsStorage.Get(databaseContext, loweredKey, throwOnConflict: false);
if (doc != null && doc.Etag <= lastIndexedEtag)
{
documents.Add(doc);
}
}
}
using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats))
{
while (docsEnumerator.MoveNext(out IEnumerable mapResults))
{
token.ThrowIfCancellationRequested();
var current = docsEnumerator.Current;
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index.Name}'. Processing document: {current.Id}.");
}
try
{
_index.HandleMap(current.LowerId, mapResults, indexWriter, indexContext, collectionStats);
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
}
}
}
if (CanContinueBatch(databaseContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, batchCount) == false)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (batchCount == 0 || batchCount >= pageSize)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing handle references for '{_index} ({_index.Name})'. Processed {count} references in '{referencedCollection.Name}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
switch (actionType)
{
case ActionType.Document:
_indexStorage.WriteLastReferenceEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
case ActionType.Tombstone:
_indexStorage.WriteLastReferenceTombstoneEtag(indexContext.Transaction, collection, referencedCollection, lastEtag);
break;
default:
throw new NotSupportedException();
}
moreWorkFound = true;
}
}
}
return(moreWorkFound);
}
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name} ({_index.IndexId})'. Collection: {collection}.");
}
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name} ({_index.IndexId})'. LastMappedEtag: {lastMappedEtag}.");
}
var lastEtag = lastMappedEtag;
var count = 0;
var resultsCount = 0;
var pageSize = int.MaxValue;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
}
var documents = GetDocumentsEnumerator(databaseContext, collection, lastEtag, pageSize);
using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats))
{
while (true)
{
IEnumerable mapResults;
if (docsEnumerator.MoveNext(out mapResults) == false)
{
collectionStats.RecordMapCompletedReason("No more documents to index");
keepRunning = false;
break;
}
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
var current = docsEnumerator.Current;
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Executing map for '{_index.Name} ({_index.IndexId})'. Processing document: {current.Key}.");
}
collectionStats.RecordMapAttempt();
count++;
lastEtag = current.Etag;
try
{
var numberOfResults = _index.HandleMap(current.LoweredKey, mapResults,
indexWriter, indexContext, collectionStats);
_index.MapsPerSec.Mark(numberOfResults);
resultsCount += numberOfResults;
collectionStats.RecordMapSuccess();
}
catch (Exception e)
{
_index.HandleError(e);
collectionStats.RecordMapError();
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Failed to execute mapping function on '{current.Key}' for '{_index.Name} ({_index.IndexId})'.",
e);
}
collectionStats.AddMapError(current.Key,
$"Failed to execute mapping function on {current.Key}. Exception: {e}");
}
if (CanContinueBatch(collectionStats, lastEtag, lastCollectionEtag) == false)
{
keepRunning = false;
break;
}
if (count >= pageSize)
{
keepRunning = false;
break;
}
if (MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name} ({_index.IndexId})'. Processed {count:#,#;;0} documents and {resultsCount:#,#;;0} map results in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedDocEtags[collection] = lastEtag;
}
moreWorkFound = true;
}
}
return(moreWorkFound);
}
public override void HandleDelete(Tombstone tombstone, string collection, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
using (DocumentIdWorker.GetSliceFromId(indexContext, tombstone.LowerId, out Slice documentIdPrefixWithTsKeySeparator, SpecialChars.RecordSeparator))
_referencesStorage.RemoveReferencesByPrefix(documentIdPrefixWithTsKeySeparator, collection, null, indexContext.Transaction);
}
public bool CanContinueBatch(DocumentsOperationContext documentsContext, TransactionOperationContext indexingContext,
IndexingStatsScope stats, IndexWriteOperation indexWriteOperation, long currentEtag, long maxEtag, int count)
{
throw new NotSupportedException();
}
public bool Execute(QueryOperationContext queryContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
var totalProcessedCount = 0;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name}'. Collection: {collection} LastMappedEtag: {lastMappedEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastMappedEtag;
var resultsCount = 0;
var pageSize = int.MaxValue;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
using (queryContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastItemEtagInCollection(queryContext, collection);
}
var items = GetItemsEnumerator(queryContext, collection, lastEtag, pageSize);
using (var itemEnumerator = _index.GetMapEnumerator(items, collection, indexContext, collectionStats, _index.Type))
{
while (true)
{
if (itemEnumerator.MoveNext(queryContext.Documents, out IEnumerable mapResults, out var etag) == false)
{
if (etag > lastEtag)
{
lastEtag = etag.Value;
}
collectionStats.RecordMapCompletedReason("No more documents to index");
keepRunning = false;
break;
}
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
var current = itemEnumerator.Current;
totalProcessedCount++;
collectionStats.RecordMapAttempt();
stats.RecordDocumentSize(current.Size);
if (_logger.IsInfoEnabled && totalProcessedCount % 8192 == 0)
{
_logger.Info($"Executing map for '{_index.Name}'. Processed count: {totalProcessedCount:#,#;;0} etag: {lastEtag:#,#;;0}.");
}
lastEtag = current.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: false);
try
{
var numberOfResults = _index.HandleMap(current, mapResults,
indexWriter, indexContext, collectionStats);
resultsCount += numberOfResults;
collectionStats.RecordMapSuccess();
_index.MapsPerSec?.MarkSingleThreaded(numberOfResults);
}
catch (Exception e) when(e.IsIndexError())
{
itemEnumerator.OnError();
_index.ErrorIndexIfCriticalException(e);
collectionStats.RecordMapError();
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
collectionStats.AddMapError(current.Id, $"Failed to execute mapping function on {current.Id}. " +
$"Exception: {e}");
}
if (CanContinueBatch(queryContext, indexContext, collectionStats, indexWriter, lastEtag, lastCollectionEtag, totalProcessedCount) == false)
{
keepRunning = false;
break;
}
if (totalProcessedCount >= pageSize)
{
keepRunning = false;
break;
}
if (MaybeRenewTransaction(queryContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
}
}
}
if (lastMappedEtag == lastEtag)
{
// the last mapped etag hasn't changed
continue;
}
moreWorkFound = true;
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executed map for '{_index.Name}' index and '{collection}' collection. Got {resultsCount:#,#;;0} map results in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_index.SaveLastState();
_indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedDocEtags[collection] = lastEtag;
}
}
}
return(moreWorkFound);
}
public override int HandleMap(LazyStringValue lowerId, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e);
}
private void HandleTreeReduction(TransactionOperationContext indexContext, IndexingStatsScope stats,
MapReduceResultsStore modifiedStore, LowLevelTransaction lowLevelTransaction,
IndexWriteOperation writer, LazyStringValue reduceKeyHash, Table table, CancellationToken token)
{
EnsureValidTreeReductionStats(stats);
var tree = modifiedStore.Tree;
var branchesToAggregate = new HashSet <long>();
var parentPagesToAggregate = new HashSet <long>();
var page = new TreePage(null, Constants.Storage.PageSize);
HashSet <long> compressedEmptyLeafs = null;
Dictionary <long, Exception> failedAggregatedLeafs = null;
foreach (var modifiedPage in modifiedStore.ModifiedPages)
{
token.ThrowIfCancellationRequested();
page.Base = lowLevelTransaction.GetPage(modifiedPage).Pointer;
stats.RecordReduceTreePageModified(page.IsLeaf);
if (page.IsLeaf == false)
{
Debug.Assert(page.IsBranch);
branchesToAggregate.Add(modifiedPage);
continue;
}
var leafPage = page;
var compressed = leafPage.IsCompressed;
if (compressed)
{
stats.RecordCompressedLeafPage();
}
using (compressed ? (DecompressedLeafPage)(leafPage = tree.DecompressPage(leafPage, skipCache: true)) : null)
{
if (leafPage.NumberOfEntries == 0)
{
if (leafPage.PageNumber == tree.State.RootPageNumber)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
var emptyPageNumber = Bits.SwapBytes(leafPage.PageNumber);
using (Slice.External(indexContext.Allocator, (byte *)&emptyPageNumber, sizeof(long), out Slice pageNumSlice))
table.DeleteByKey(pageNumSlice);
continue;
}
if (compressed)
{
// it doesn't have any entries after decompression because
// each compressed entry has the delete tombstone
if (compressedEmptyLeafs == null)
{
compressedEmptyLeafs = new HashSet <long>();
}
compressedEmptyLeafs.Add(leafPage.PageNumber);
continue;
}
throw new UnexpectedReduceTreePageException(
$"Encountered empty page which isn't a root. Page {leafPage} in '{tree.Name}' tree.");
}
var parentPage = tree.GetParentPageOf(leafPage);
stats.RecordReduceAttempts(leafPage.NumberOfEntries);
try
{
using (var result = AggregateLeafPage(leafPage, lowLevelTransaction, indexContext, token))
{
if (parentPage == -1)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
}
else
{
StoreAggregationResult(leafPage, table, result);
parentPagesToAggregate.Add(parentPage);
}
_metrics.MapReduceIndexes.ReducedPerSec.Mark(leafPage.NumberOfEntries);
stats.RecordReduceSuccesses(leafPage.NumberOfEntries);
}
}
catch (Exception e) when(e is OperationCanceledException == false)
{
if (failedAggregatedLeafs == null)
{
failedAggregatedLeafs = new Dictionary <long, Exception>();
}
failedAggregatedLeafs.Add(leafPage.PageNumber, e);
_index.ErrorIndexIfCriticalException(e);
HandleReductionError(e, reduceKeyHash, writer, stats, updateStats: parentPage == -1, page: leafPage);
}
}
}
long tmp = 0;
using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice))
{
foreach (var freedPage in modifiedStore.FreedPages)
{
tmp = Bits.SwapBytes(freedPage);
table.DeleteByKey(pageNumberSlice);
}
}
while (parentPagesToAggregate.Count > 0 || branchesToAggregate.Count > 0)
{
token.ThrowIfCancellationRequested();
var branchPages = parentPagesToAggregate;
parentPagesToAggregate = new HashSet <long>();
foreach (var pageNumber in branchPages)
{
page.Base = lowLevelTransaction.GetPage(pageNumber).Pointer;
try
{
if (page.IsBranch == false)
{
throw new UnexpectedReduceTreePageException("Parent page was found that wasn't a branch, error at " + page);
}
stats.RecordReduceAttempts(page.NumberOfEntries);
var parentPage = tree.GetParentPageOf(page);
using (var result = AggregateBranchPage(page, table, indexContext, branchesToAggregate, compressedEmptyLeafs, failedAggregatedLeafs, token))
{
if (parentPage == -1)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
}
else
{
parentPagesToAggregate.Add(parentPage);
StoreAggregationResult(page, table, result);
}
_metrics.MapReduceIndexes.ReducedPerSec.Mark(page.NumberOfEntries);
stats.RecordReduceSuccesses(page.NumberOfEntries);
}
}
catch (Exception e) when(e is OperationCanceledException == false)
{
_index.ErrorIndexIfCriticalException(e);
HandleReductionError(e, reduceKeyHash, writer, stats, updateStats: true, page: page);
}
finally
{
branchesToAggregate.Remove(pageNumber);
}
}
if (parentPagesToAggregate.Count == 0 && branchesToAggregate.Count > 0)
{
// we still have unaggregated branches which were modified but their children were not modified (branch page splitting) so we missed them
parentPagesToAggregate.Add(branchesToAggregate.First());
}
}
if (compressedEmptyLeafs != null && compressedEmptyLeafs.Count > 0)
{
// we had some compressed pages that are empty after decompression
// let's remove them and reduce the tree once again
modifiedStore.ModifiedPages.Clear();
modifiedStore.FreedPages.Clear();
foreach (var pageNumber in compressedEmptyLeafs)
{
page.Base = lowLevelTransaction.GetPage(pageNumber).Pointer;
using (var emptyPage = tree.DecompressPage(page, skipCache: true))
{
if (emptyPage.NumberOfEntries > 0) // could be changed meanwhile
{
continue;
}
modifiedStore.Tree.RemoveEmptyDecompressedPage(emptyPage);
}
}
HandleTreeReduction(indexContext, stats, modifiedStore, lowLevelTransaction, writer, reduceKeyHash, table, token);
}
}
public override void HandleDelete(Tombstone tombstone, string collection, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
StaticIndexHelper.HandleReferencesDelete(_handleReferences, _handleCompareExchangeReferences, tombstone, collection, writer, indexContext, stats);
base.HandleDelete(tombstone, collection, writer, indexContext, stats);
}
private void HandleTreeReduction(TransactionOperationContext indexContext, IndexingStatsScope stats,
CancellationToken token, MapReduceResultsStore modifiedStore, LowLevelTransaction lowLevelTransaction,
IndexWriteOperation writer, LazyStringValue reduceKeyHash, Table table)
{
EnsureValidTreeReductionStats(stats);
var tree = modifiedStore.Tree;
var branchesToAggregate = new HashSet <long>();
var parentPagesToAggregate = new HashSet <long>();
var page = new TreePage(null, Constants.Storage.PageSize);
foreach (var modifiedPage in modifiedStore.ModifiedPages)
{
token.ThrowIfCancellationRequested();
page.Base = lowLevelTransaction.GetPage(modifiedPage).Pointer;
stats.RecordReduceTreePageModified(page.IsLeaf);
if (page.IsLeaf == false)
{
Debug.Assert(page.IsBranch);
branchesToAggregate.Add(modifiedPage);
continue;
}
var leafPage = page;
var compressed = leafPage.IsCompressed;
if (compressed)
{
stats.RecordCompressedLeafPage();
}
using (compressed ? (DecompressedLeafPage)(leafPage = tree.DecompressPage(leafPage, skipCache: true)) : null)
{
if (leafPage.NumberOfEntries == 0)
{
if (leafPage.PageNumber != tree.State.RootPageNumber)
{
throw new InvalidOperationException(
$"Encountered empty page which isn't a root. Page #{leafPage.PageNumber} in '{tree.Name}' tree.");
}
writer.DeleteReduceResult(reduceKeyHash, stats);
var emptyPageNumber = Bits.SwapBytes(leafPage.PageNumber);
using (Slice.External(indexContext.Allocator, (byte *)&emptyPageNumber, sizeof(long), out Slice pageNumSlice))
table.DeleteByKey(pageNumSlice);
continue;
}
var parentPage = tree.GetParentPageOf(leafPage);
stats.RecordReduceAttempts(leafPage.NumberOfEntries);
try
{
using (var result = AggregateLeafPage(leafPage, lowLevelTransaction, indexContext, token))
{
if (parentPage == -1)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
}
else
{
StoreAggregationResult(leafPage.PageNumber, leafPage.NumberOfEntries, table, result);
parentPagesToAggregate.Add(parentPage);
}
_metrics.MapReduceReducedPerSecond.Mark(leafPage.NumberOfEntries);
stats.RecordReduceSuccesses(leafPage.NumberOfEntries);
}
}
catch (Exception e)
{
_index.HandleError(e);
var message =
$"Failed to execute reduce function for reduce key '{tree.Name}' on a leaf page #{leafPage} of '{_indexDefinition.Name}' index.";
if (_logger.IsInfoEnabled)
{
_logger.Info(message, e);
}
if (parentPage == -1)
{
stats.RecordReduceErrors(leafPage.NumberOfEntries);
stats.AddReduceError(message + $" Exception: {e}");
}
}
}
}
long tmp = 0;
using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice))
{
foreach (var freedPage in modifiedStore.FreedPages)
{
tmp = Bits.SwapBytes(freedPage);
table.DeleteByKey(pageNumberSlice);
}
}
while (parentPagesToAggregate.Count > 0 || branchesToAggregate.Count > 0)
{
token.ThrowIfCancellationRequested();
var branchPages = parentPagesToAggregate;
parentPagesToAggregate = new HashSet <long>();
foreach (var pageNumber in branchPages)
{
page.Base = lowLevelTransaction.GetPage(pageNumber).Pointer;
try
{
if (page.IsBranch == false)
{
throw new InvalidOperationException("Parent page was found that wasn't a branch, error at " +
page.PageNumber);
}
stats.RecordReduceAttempts(page.NumberOfEntries);
var parentPage = tree.GetParentPageOf(page);
using (var result = AggregateBranchPage(page, table, indexContext, branchesToAggregate, token))
{
if (parentPage == -1)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
}
else
{
parentPagesToAggregate.Add(parentPage);
StoreAggregationResult(page.PageNumber, page.NumberOfEntries, table, result);
}
_metrics.MapReduceReducedPerSecond.Mark(page.NumberOfEntries);
stats.RecordReduceSuccesses(page.NumberOfEntries);
}
}
catch (Exception e)
{
_index.HandleError(e);
var message =
$"Failed to execute reduce function for reduce key '{tree.Name}' on a branch page #{page} of '{_indexDefinition.Name}' index.";
if (_logger.IsInfoEnabled)
{
_logger.Info(message, e);
}
stats.RecordReduceErrors(page.NumberOfEntries);
stats.AddReduceError(message + $" Exception: {e}");
}
finally
{
branchesToAggregate.Remove(pageNumber);
}
}
if (parentPagesToAggregate.Count == 0 && branchesToAggregate.Count > 0)
{
// we still have unaggregated branches which were modified but their children were not modified (branch page splitting) so we missed them
parentPagesToAggregate.Add(branchesToAggregate.First());
}
}
}
public override void HandleDelete(Tombstone tombstone, string collection, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
throw new NotImplementedException();
}
