public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name}'. Collection: {collection} LastMappedEtag: {lastMappedEtag}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastMappedEtag;
var count = 0;
var resultsCount = 0;
var pageSize = int.MaxValue;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastDocumentEtagInCollection(databaseContext, collection);
}
var documents = GetDocumentsEnumerator(databaseContext, collection, lastEtag, pageSize);
using (var docsEnumerator = _index.GetMapEnumerator(documents, collection, indexContext, collectionStats))
{
while (true)
{
if (docsEnumerator.MoveNext(out IEnumerable mapResults) == false)
{
collectionStats.RecordMapCompletedReason("No more documents to index");
keepRunning = false;
break;
}
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
var current = docsEnumerator.Current;
count++;
collectionStats.RecordMapAttempt();
stats.RecordDocumentSize(current.Data.Size);
if (_logger.IsInfoEnabled && count % 8192 == 0)
{
_logger.Info($"Executing map for '{_index.Name}'. Proccessed count: {count:#,#;;0} etag: {lastEtag}.");
}
lastEtag = current.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombsone: false);
try
{
var numberOfResults = _index.HandleMap(current.LowerId, mapResults,
indexWriter, indexContext, collectionStats);
_index.MapsPerSec.Mark(numberOfResults);
resultsCount += numberOfResults;
collectionStats.RecordMapSuccess();
}
catch (Exception e)
{
docsEnumerator.OnError();
_index.ThrowIfCorruptionException(e);
collectionStats.RecordMapError();
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
collectionStats.AddMapError(current.Id, $"Failed to execute mapping function on {current.Id}. " +
$"Exception: {e}");
}
if (CanContinueBatch(databaseContext, indexContext, collectionStats, lastEtag, lastCollectionEtag, count) == false)
{
keepRunning = false;
break;
}
if (count >= pageSize)
{
keepRunning = false;
break;
}
if (MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Done executing map for '{_index.Name}'. Proccessed count: {count:#,#;;0} LastEtag {lastEtag}.");
}
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name}'. Processed {count:#,#;;0} documents and {resultsCount:#,#;;0} map results in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedDocEtags[collection] = lastEtag;
}
moreWorkFound = true;
}
}
return(moreWorkFound);
}
public Enumerator(IEnumerator enumerator, AnonymousObjectToBlittableMapResultsEnumerableWrapper parent, IndexingStatsScope createBlittableResult)
{
_enumerator = enumerator;
_parent = parent;
_createBlittableResult = createBlittableResult;
_groupByFields = _parent._groupByFields;
_reduceKeyProcessor = _parent._reduceKeyProcessor;
}
private void HandleNestedValuesReduction(TransactionOperationContext indexContext, IndexingStatsScope stats,
CancellationToken token, MapReduceResultsStore modifiedStore,
IndexWriteOperation writer, LazyStringValue reduceKeyHash)
{
EnsureValidNestedValuesReductionStats(stats);
var numberOfEntriesToReduce = 0;
try
{
var section = modifiedStore.GetNestedResultsSection();
if (section.IsModified == false)
{
return;
}
using (_nestedValuesReductionStats.NestedValuesRead.Start())
{
numberOfEntriesToReduce += section.GetResults(indexContext, _aggregationBatch.Items);
}
stats.RecordReduceAttempts(numberOfEntriesToReduce);
AggregationResult result;
using (_nestedValuesReductionStats.NestedValuesAggregation.Start())
{
result = AggregateOn(_aggregationBatch.Items, indexContext, token);
}
if (section.IsNew == false)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
}
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
_index.ReducesPerSec.Mark(numberOfEntriesToReduce);
_metrics.MapReduceIndexes.ReducedPerSec.Mark(numberOfEntriesToReduce);
stats.RecordReduceSuccesses(numberOfEntriesToReduce);
}
catch (Exception e)
{
_index.ThrowIfCorruptionException(e);
LogReductionError(e, reduceKeyHash, stats, updateStats: true, page: null, numberOfNestedValues: numberOfEntriesToReduce);
}
}
public override void HandleDelete(Tombstone tombstone, string collection, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
StaticIndexHelper.HandleReferencesDelete(_handleReferences, _handleCompareExchangeReferences, tombstone, collection, writer, indexContext, stats);
base.HandleDelete(tombstone, collection, writer, indexContext, stats);
}
public override int HandleMap(IndexItem indexItem, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
if (_enumerationWrappers.TryGetValue(CurrentIndexingScope.Current.SourceCollection, out AnonymousObjectToBlittableMapResultsEnumerableWrapper wrapper) == false)
{
_enumerationWrappers[CurrentIndexingScope.Current.SourceCollection] = wrapper = new AnonymousObjectToBlittableMapResultsEnumerableWrapper(this, indexContext);
}
wrapper.InitializeForEnumeration(mapResults, indexContext, stats);
return(PutMapResults(indexItem.LowerId, indexItem.Id, wrapper, indexContext, stats));
}
public override IIndexedDocumentsEnumerator GetMapEnumerator(IEnumerable <Document> documents, string collection, TransactionOperationContext indexContext, IndexingStatsScope stats, IndexType type)
{
return(new StaticIndexDocsEnumerator(documents, _compiled.Maps[collection], collection, stats, type));
}
public override void HandleDelete(Tombstone tombstone, string collection, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e);
}
private static async Task ActualTest(int numberOfUsers, string[] locations, Index index,
MapReduceIndexingContext mapReduceContext, IIndexingWork reducer, DocumentDatabase database)
{
TransactionOperationContext indexContext;
using (index._contextPool.AllocateOperationContext(out indexContext))
{
ulong hashOfReduceKey = 73493;
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < numberOfUsers; i++)
{
using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
{
["Count"] = 1,
["Location"] = locations[i % locations.Length]
}, $"users/{i}"))
{
store.Add(i, mappedResult);
}
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, null));
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
tx.Commit();
}
using (var termSingleUse = DocumentsOperationContext.ShortTermSingleUse(database))
{
var queryResult = await
index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"),
termSingleUse,
OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected, results[i].Data["Count"]);
}
}
// update
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < locations.Length; i++)
{
using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
{
["Count"] = 2, // increased by 1
["Location"] = locations[i % locations.Length]
}, $"users/{i}"))
{
store.Add(i, mappedResult);
}
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, null));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
tx.Commit();
}
using (var shortTermSingleUse = DocumentsOperationContext.ShortTermSingleUse(database))
{
var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"),
shortTermSingleUse,
OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected + 1, results[i].Data["Count"]);
}
}
// delete
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < locations.Length; i++)
{
store.Delete(i);
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, null));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
tx.Commit();
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
}
using (var documentsOperationContext = DocumentsOperationContext.ShortTermSingleUse(database))
{
var queryResult = await index.Query(new IndexQueryServerSide("FROM Users ORDER BY Location"),
documentsOperationContext,
OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected - 1, results[i].Data["Count"]);
}
}
}
}
public async Task CleanupOfMultiMapIndexWithLoadDocument()
{
var indexDefinition = new IndexDefinition()
{
Name = "NewIndex",
Maps = new HashSet <string>
{
"from p in docs.Orders select new { CompanyName = LoadDocument(p.Company, \"Companies\").Name }",
"from p in docs.Companies select new { CompanyName = p.Name }"
}
};
using (var database = CreateDocumentDatabase())
using (var index = MapIndex.CreateNew(indexDefinition, database))
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "key/1", new DynamicJsonValue
{
["Name"] = "John",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Orders"
}
}))
{
database.DocumentsStorage.Put(context, "key/1", null, doc);
}
tx.Commit();
}
var batchStats = new IndexingRunStats();
var stats = new IndexingStatsScope(batchStats);
index.DoIndexingWork(stats, CancellationToken.None);
var tombstones = index.GetLastProcessedTombstonesPerCollection(ITombstoneAware.TombstoneType.Documents);
Assert.Equal(2, tombstones.Count);
Assert.Equal(0, tombstones["Orders"]);
Assert.Equal(0, tombstones["Companies"]);
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Orders", 0, 0, 128).Count();
Assert.Equal(0, count);
}
using (var tx = context.OpenWriteTransaction())
{
database.DocumentsStorage.Delete(context, "key/1", null);
tx.Commit();
}
tombstones = index.GetLastProcessedTombstonesPerCollection(ITombstoneAware.TombstoneType.Documents);
Assert.Equal(2, tombstones.Count);
Assert.Equal(0, tombstones["Orders"]);
Assert.Equal(0, tombstones["Companies"]);
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Orders", 0, 0, 128).Count();
Assert.Equal(1, count);
}
await database.TombstoneCleaner.ExecuteCleanup();
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Orders", 0, 0, 128).Count();
Assert.Equal(1, count);
}
batchStats = new IndexingRunStats();
stats = new IndexingStatsScope(batchStats);
index.DoIndexingWork(stats, CancellationToken.None);
tombstones = index.GetLastProcessedTombstonesPerCollection(ITombstoneAware.TombstoneType.Documents);
Assert.Equal(2, tombstones.Count);
Assert.Equal(2, tombstones["Orders"]);
Assert.Equal(0, tombstones["Companies"]);
await database.TombstoneCleaner.ExecuteCleanup();
using (context.OpenReadTransaction())
{
var list = database.DocumentsStorage.GetTombstonesFrom(context, "Orders", 0, 0, 128).ToList();
Assert.Equal(0, list.Count);
}
}
}
private void HandleReductionError(Exception error, LazyStringValue reduceKeyHash, IndexWriteOperation writer, IndexingStatsScope stats, bool updateStats, TreePage page,
int numberOfNestedValues = -1)
{
var builder = new StringBuilder("Failed to execute reduce function on ");
if (page != null)
{
builder.Append($"page {page} ");
}
else
{
builder.Append("nested values ");
}
builder.Append($"of '{_indexDefinition.Name}' index. The relevant reduce result is going to be removed from the index ");
builder.Append($"as it would be incorrect due to encountered errors (reduce key hash: {reduceKeyHash}");
var sampleItem = _aggregationBatch?.Items?.FirstOrDefault();
if (sampleItem != null)
{
builder.Append($", sample item to reduce: {sampleItem}");
}
builder.Append(")");
var message = builder.ToString();
if (_logger.IsInfoEnabled)
{
_logger.Info(message, error);
}
try
{
writer.DeleteReduceResult(reduceKeyHash, stats);
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to delete an index result from '${_indexDefinition.Name}' index on reduce error (reduce key hash: ${reduceKeyHash})", e);
}
}
if (updateStats)
{
var numberOfEntries = page?.NumberOfEntries ?? numberOfNestedValues;
Debug.Assert(numberOfEntries != -1);
// we'll only want to record exceptions on some of these, to give the
// user information about what is going on, otherwise we'll have to wait a lot if we
// are processing a big batch, and this can be a perf killer. See: RavenDB-11038
stats.RecordReduceErrors(numberOfEntries);
if (stats.NumberOfKeptReduceErrors < IndexStorage.MaxNumberOfKeptErrors)
{
stats.AddReduceError(message + $" Exception: {error}");
}
var failureInfo = new IndexFailureInformation
{
Name = _index.Name,
MapErrors = stats.MapErrors,
MapAttempts = stats.MapAttempts,
ReduceErrors = stats.ReduceErrors,
ReduceAttempts = stats.ReduceAttempts
};
if (failureInfo.IsInvalidIndex(_isStaleBecauseOfRunningReduction))
{
throw new ExcessiveNumberOfReduceErrorsException("Excessive number of errors during the reduce phase for the current batch. Failure info: " +
failureInfo.GetErrorMessage());
}
}
}
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, Lazy <IndexWriteOperation> writeOperation,
IndexingStatsScope stats, CancellationToken token)
{
if (_mapReduceContext.StoreByReduceKeyHash.Count == 0)
{
WriteLastEtags(indexContext); // we need to write etags here, because if we filtered everything during map then we will loose last indexed etag information and this will cause an endless indexing loop
return(false);
}
ReduceResultsSchema.Create(indexContext.Transaction.InnerTransaction, PageNumberToReduceResultTableName, 32);
var table = indexContext.Transaction.InnerTransaction.OpenTable(ReduceResultsSchema, PageNumberToReduceResultTableName);
var lowLevelTransaction = indexContext.Transaction.InnerTransaction.LowLevelTransaction;
var writer = writeOperation.Value;
var treeScopeStats = stats.For(IndexingOperation.Reduce.TreeScope, start: false);
var nestedValuesScopeStats = stats.For(IndexingOperation.Reduce.NestedValuesScope, start: false);
foreach (var store in _mapReduceContext.StoreByReduceKeyHash)
{
token.ThrowIfCancellationRequested();
using (var reduceKeyHash = indexContext.GetLazyString(store.Key.ToString(CultureInfo.InvariantCulture)))
using (store.Value)
using (_aggregationBatch)
{
var modifiedStore = store.Value;
switch (modifiedStore.Type)
{
case MapResultsStorageType.Tree:
using (treeScopeStats.Start())
{
HandleTreeReduction(indexContext, treeScopeStats, modifiedStore, lowLevelTransaction, writer, reduceKeyHash, table, token);
}
break;
case MapResultsStorageType.Nested:
using (nestedValuesScopeStats.Start())
{
HandleNestedValuesReduction(indexContext, nestedValuesScopeStats, modifiedStore, writer, reduceKeyHash, token);
}
break;
default:
throw new ArgumentOutOfRangeException(modifiedStore.Type.ToString());
}
}
if (_mapReduceContext.FreedPages.Count > 0)
{
long tmp = 0;
using (treeScopeStats.Start())
using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice))
{
foreach (var freedPage in _mapReduceContext.FreedPages)
{
tmp = Bits.SwapBytes(freedPage);
table.DeleteByKey(pageNumberSlice);
}
}
}
}
if (stats.Duration >= MinReduceDurationToCalculateProcessMemoryUsage)
{
var workingSet = MemoryInformation.GetWorkingSetInBytes();
var privateMemory = MemoryInformation.GetManagedMemoryInBytes() + MemoryInformation.GetUnManagedAllocationsInBytes();
stats.RecordReduceMemoryStats(workingSet, privateMemory);
}
WriteLastEtags(indexContext);
return(false);
}
protected abstract AggregationResult AggregateOn(List <BlittableJsonReaderObject> aggregationBatch, TransactionOperationContext indexContext, IndexingStatsScope stats, CancellationToken token);
private AggregationResult AggregateBatchResults(List <BlittableJsonReaderObject> aggregationBatch, TransactionOperationContext indexContext, IndexingStatsScope stats, CancellationToken token)
{
AggregationResult result;
try
{
result = AggregateOn(aggregationBatch, indexContext, stats, token);
}
finally
{
aggregationBatch.Clear();
}
return(result);
}
private void HandleTreeReduction(TransactionOperationContext indexContext, IndexingStatsScope stats,
MapReduceResultsStore modifiedStore, LowLevelTransaction lowLevelTransaction,
IndexWriteOperation writer, LazyStringValue reduceKeyHash, Table table, CancellationToken token)
{
if (modifiedStore.ModifiedPages.Count == 0)
{
return;
}
EnsureValidTreeReductionStats(stats);
var tree = modifiedStore.Tree;
var branchesToAggregate = new HashSet <long>();
var parentPagesToAggregate = new HashSet <long>();
var page = new TreePage(null, Constants.Storage.PageSize);
HashSet <long> compressedEmptyLeafs = null;
Dictionary <long, Exception> failedAggregatedLeafs = null;
foreach (var modifiedPage in modifiedStore.ModifiedPages)
{
token.ThrowIfCancellationRequested();
page.Base = lowLevelTransaction.GetPage(modifiedPage).Pointer;
stats.RecordReduceTreePageModified(page.IsLeaf);
if (page.IsLeaf == false)
{
Debug.Assert(page.IsBranch);
branchesToAggregate.Add(modifiedPage);
continue;
}
var leafPage = page;
var compressed = leafPage.IsCompressed;
if (compressed)
{
stats.RecordCompressedLeafPage();
}
using (compressed ? (DecompressedLeafPage)(leafPage = tree.DecompressPage(leafPage, skipCache: true)) : null)
{
if (leafPage.NumberOfEntries == 0)
{
if (leafPage.PageNumber == tree.State.RootPageNumber)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
var emptyPageNumber = Bits.SwapBytes(leafPage.PageNumber);
using (Slice.External(indexContext.Allocator, (byte *)&emptyPageNumber, sizeof(long), out Slice pageNumSlice))
table.DeleteByKey(pageNumSlice);
continue;
}
if (compressed)
{
// it doesn't have any entries after decompression because
// each compressed entry has the delete tombstone
if (compressedEmptyLeafs == null)
{
compressedEmptyLeafs = new HashSet <long>();
}
compressedEmptyLeafs.Add(leafPage.PageNumber);
continue;
}
throw new UnexpectedReduceTreePageException(
$"Encountered empty page which isn't a root. Page {leafPage} in '{tree.Name}' tree.");
}
var parentPage = tree.GetParentPageOf(leafPage);
stats.RecordReduceAttempts(leafPage.NumberOfEntries);
try
{
using (var result = AggregateLeafPage(leafPage, lowLevelTransaction, indexContext, token))
{
if (parentPage == -1)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
}
else
{
StoreAggregationResult(leafPage, table, result);
parentPagesToAggregate.Add(parentPage);
}
_index.ReducesPerSec.MarkSingleThreaded(leafPage.NumberOfEntries);
_metrics.MapReduceIndexes.ReducedPerSec.Mark(leafPage.NumberOfEntries);
stats.RecordReduceSuccesses(leafPage.NumberOfEntries);
}
}
catch (Exception e) when(e.IsOperationCanceled() == false && e.IsOutOfMemory() == false)
{
if (failedAggregatedLeafs == null)
{
failedAggregatedLeafs = new Dictionary <long, Exception>();
}
failedAggregatedLeafs.Add(leafPage.PageNumber, e);
_index.ErrorIndexIfCriticalException(e);
HandleReductionError(e, reduceKeyHash, writer, stats, updateStats: parentPage == -1, page: leafPage);
}
}
}
while (parentPagesToAggregate.Count > 0 || branchesToAggregate.Count > 0)
{
token.ThrowIfCancellationRequested();
var branchPages = parentPagesToAggregate;
parentPagesToAggregate = new HashSet <long>();
foreach (var pageNumber in branchPages)
{
page.Base = lowLevelTransaction.GetPage(pageNumber).Pointer;
try
{
if (page.IsBranch == false)
{
throw new UnexpectedReduceTreePageException("Parent page was found that wasn't a branch, error at " + page);
}
stats.RecordReduceAttempts(page.NumberOfEntries);
var parentPage = tree.GetParentPageOf(page);
using (var result = AggregateBranchPage(page, table, indexContext, branchesToAggregate, compressedEmptyLeafs, failedAggregatedLeafs, tree, token))
{
if (parentPage == -1)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
}
else
{
parentPagesToAggregate.Add(parentPage);
StoreAggregationResult(page, table, result);
}
_index.ReducesPerSec.MarkSingleThreaded(page.NumberOfEntries);
_metrics.MapReduceIndexes.ReducedPerSec.Mark(page.NumberOfEntries);
stats.RecordReduceSuccesses(page.NumberOfEntries);
}
}
catch (Exception e) when(e.IsOperationCanceled() == false && e.IsOutOfMemory() == false)
{
_index.ErrorIndexIfCriticalException(e);
HandleReductionError(e, reduceKeyHash, writer, stats, updateStats: true, page: page);
}
finally
{
branchesToAggregate.Remove(pageNumber);
}
}
if (parentPagesToAggregate.Count == 0 && branchesToAggregate.Count > 0)
{
// we still have unaggregated branches which were modified but their children were not modified (branch page splitting) so we missed them
parentPagesToAggregate.Add(branchesToAggregate.First());
}
_index.UpdateThreadAllocations(indexContext, writer, stats, updateReduceStats: true);
}
if (compressedEmptyLeafs != null && compressedEmptyLeafs.Count > 0)
{
// we had some compressed pages that are empty after decompression
// let's remove them and reduce the tree once again
modifiedStore.ModifiedPages.Clear();
foreach (var pageNumber in compressedEmptyLeafs)
{
page.Base = lowLevelTransaction.GetPage(pageNumber).Pointer;
using (var emptyPage = tree.DecompressPage(page, skipCache: true))
{
if (emptyPage.NumberOfEntries > 0) // could be changed meanwhile
{
continue;
}
modifiedStore.Tree.RemoveEmptyDecompressedPage(emptyPage);
}
}
HandleTreeReduction(indexContext, stats, modifiedStore, lowLevelTransaction, writer, reduceKeyHash, table, token);
}
}
[InlineData(10)] // nested section
public void Getting_trees_for_multiple_docs(int numberOfDocs)
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition
{
Etag = 1,
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
var docIds = Enumerable.Range(0, numberOfDocs).Select(x => x % 2 == 0 ? "orders/" + x : "Orders/" + x).ToArray();
IEnumerable <ReduceTree> trees;
using (index.GetReduceTree(docIds, out trees))
{
var result = trees.ToList();
Assert.Equal(2, result.Count);
for (int i = 0; i < 2; i++)
{
var tree = result[0];
List <ReduceTreePage> pages;
if (tree.Depth > 1)
{
// real tree
pages = tree.Root.Children;
}
else
{
// nested section
pages = new List <ReduceTreePage>
{
tree.Root
};
}
Assert.NotNull(tree.Root.AggregationResult);
var seenSources = new HashSet <string>();
foreach (var leafPage in pages)
{
foreach (var entry in leafPage.Entries)
{
Assert.NotNull(entry.Source);
seenSources.Add(entry.Source);
}
}
Assert.Equal(numberOfDocs, seenSources.Count);
Assert.Equal(numberOfDocs, pages.Sum(x => x.Entries.Count));
}
}
}
}
}
}
public StaticIndexDocsEnumerator(IEnumerable <Document> docs, List <IndexingFunc> funcs, string collection, IndexingStatsScope stats, IndexType type)
: this(docs)
{
_documentReadStats = stats?.For(IndexingOperation.Map.DocumentRead, start: false);
var indexingFunctionType = type.IsJavaScript() ? IndexingOperation.Map.Jint : IndexingOperation.Map.Linq;
var mapFuncStats = stats?.For(indexingFunctionType, start: false);
if (funcs.Count == 1)
{
_resultsOfCurrentDocument =
new TimeCountingEnumerable(funcs[0](new DynamicIteratonOfCurrentDocumentWrapper(this)), mapFuncStats);
}
else
{
_multipleIndexingFunctionsEnumerator = new MultipleIndexingFunctionsEnumerator(funcs, new DynamicIteratonOfCurrentDocumentWrapper(this));
_resultsOfCurrentDocument = new TimeCountingEnumerable(_multipleIndexingFunctionsEnumerator, mapFuncStats);
}
CurrentIndexingScope.Current.SetSourceCollection(collection, mapFuncStats);
}
public override void HandleDelete(Tombstone tombstone, string collection, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
if (_referencedCollections.Count > 0)
{
_handleReferences.HandleDelete(tombstone, collection, writer, indexContext, stats);
}
base.HandleDelete(tombstone, collection, writer, indexContext, stats);
}
public bool Execute(DocumentsOperationContext databaseContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
var pageSize = int.MaxValue;
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index.Name} ({_index.IndexId})'. Collection: {collection}.");
}
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
var lastTombstoneEtag = _indexStorage.ReadLastProcessedTombstoneEtag(indexContext.Transaction,
collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. LastMappedEtag: {lastMappedEtag}. LastTombstoneEtag: {lastTombstoneEtag}.");
}
var lastEtag = lastTombstoneEtag;
var count = 0;
var sw = new Stopwatch();
IndexWriteOperation indexWriter = null;
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
var batchCount = 0;
using (databaseContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastTombstoneEtagInCollection(databaseContext, collection);
}
var tombstones = collection == Constants.Indexing.AllDocumentsCollection
? _documentsStorage.GetTombstonesFrom(databaseContext, lastEtag + 1, 0, pageSize)
: _documentsStorage.GetTombstonesFrom(databaseContext, collection, lastEtag + 1, 0, pageSize);
foreach (var tombstone in tombstones)
{
token.ThrowIfCancellationRequested();
if (indexWriter == null)
{
indexWriter = writeOperation.Value;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processing tombstone {tombstone.Key} ({tombstone.Etag}).");
}
count++;
batchCount++;
lastEtag = tombstone.Etag;
if (tombstone.DeletedEtag > lastMappedEtag)
{
continue; // no-op, we have not yet indexed this document
}
_index.HandleDelete(tombstone, collection, indexWriter, indexContext, collectionStats);
if (CanContinueBatch(collectionStats, lastEtag, lastCollectionEtag) == false)
{
keepRunning = false;
break;
}
if (MapDocuments.MaybeRenewTransaction(databaseContext, sw, _configuration, ref maxTimeForDocumentTransactionToRemainOpen))
{
break;
}
}
if (batchCount == 0 || batchCount >= pageSize)
{
break;
}
}
}
if (count == 0)
{
continue;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing cleanup for '{_index} ({_index.Name})'. Processed {count} tombstones in '{collection}' collection in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_indexStorage.WriteLastTombstoneEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedTombstoneEtags[collection] = lastEtag;
}
moreWorkFound = true;
}
}
return(moreWorkFound);
}
public override IIndexedDocumentsEnumerator GetMapEnumerator(IEnumerable <Document> documents, string collection, TransactionOperationContext indexContext, IndexingStatsScope stats, IndexType type)
{
throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e);
}
public (bool MoreWorkFound, Index.CanContinueBatchResult BatchContinuationResult) Execute(QueryOperationContext queryContext, TransactionOperationContext indexContext,
Lazy <IndexWriteOperation> writeOperation, IndexingStatsScope stats, CancellationToken token)
{
var maxTimeForDocumentTransactionToRemainOpen = Debugger.IsAttached == false
? _configuration.MaxTimeForDocumentTransactionToRemainOpen.AsTimeSpan
: TimeSpan.FromMinutes(15);
var moreWorkFound = false;
var batchContinuationResult = Index.CanContinueBatchResult.None;
var totalProcessedCount = 0;
foreach (var collection in _index.Collections)
{
using (var collectionStats = stats.For("Collection_" + collection))
{
var lastMappedEtag = _indexStorage.ReadLastIndexedEtag(indexContext.Transaction, collection);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executing map for '{_index.Name}'. Collection: {collection} LastMappedEtag: {lastMappedEtag:#,#;;0}.");
}
var inMemoryStats = _index.GetStats(collection);
var lastEtag = lastMappedEtag;
var resultsCount = 0;
var pageSize = int.MaxValue;
var sw = new Stopwatch();
var keepRunning = true;
var lastCollectionEtag = -1L;
while (keepRunning)
{
using (queryContext.OpenReadTransaction())
{
sw.Restart();
if (lastCollectionEtag == -1)
{
lastCollectionEtag = _index.GetLastItemEtagInCollection(queryContext, collection);
}
var items = GetItemsEnumerator(queryContext, collection, lastEtag, pageSize);
using (var itemEnumerator = _index.GetMapEnumerator(items, collection, indexContext, collectionStats, _index.Type))
{
while (true)
{
if (itemEnumerator.MoveNext(queryContext.Documents, out IEnumerable mapResults, out var etag) == false)
{
if (etag > lastEtag)
{
lastEtag = etag.Value;
}
collectionStats.RecordBatchCompletedReason(IndexingWorkType.Map, "No more documents to index");
keepRunning = false;
break;
}
token.ThrowIfCancellationRequested();
var current = itemEnumerator.Current;
totalProcessedCount++;
collectionStats.RecordMapAttempt();
stats.RecordDocumentSize(current.Size);
if (_logger.IsInfoEnabled && totalProcessedCount % 8192 == 0)
{
_logger.Info($"Executing map for '{_index.Name}'. Processed count: {totalProcessedCount:#,#;;0} etag: {lastEtag:#,#;;0}.");
}
lastEtag = current.Etag;
inMemoryStats.UpdateLastEtag(lastEtag, isTombstone: false);
try
{
current.SkipLuceneDelete = _indexStorage.LowerThanLastDatabaseEtagOnIndexCreation(current.Etag);
var numberOfResults = _index.HandleMap(current, mapResults,
writeOperation, indexContext, collectionStats);
resultsCount += numberOfResults;
collectionStats.RecordMapSuccess();
_index.MapsPerSec?.MarkSingleThreaded(numberOfResults);
}
catch (Exception e) when(e.IsIndexError())
{
itemEnumerator.OnError();
_index.ErrorIndexIfCriticalException(e);
collectionStats.RecordMapError();
if (_logger.IsInfoEnabled)
{
_logger.Info($"Failed to execute mapping function on '{current.Id}' for '{_index.Name}'.", e);
}
collectionStats.AddMapError(current.Id, $"Failed to execute mapping function on {current.Id}. " +
$"Exception: {e}");
}
var parameters = new CanContinueBatchParameters(collectionStats, IndexingWorkType.Map, queryContext, indexContext, writeOperation,
lastEtag, lastCollectionEtag, totalProcessedCount, sw);
batchContinuationResult = _index.CanContinueBatch(in parameters, ref maxTimeForDocumentTransactionToRemainOpen);
if (batchContinuationResult != Index.CanContinueBatchResult.True)
{
keepRunning = batchContinuationResult == Index.CanContinueBatchResult.RenewTransaction;
break;
}
if (totalProcessedCount >= pageSize)
{
keepRunning = false;
break;
}
}
}
}
}
if (lastMappedEtag == lastEtag)
{
// the last mapped etag hasn't changed
continue;
}
moreWorkFound = true;
if (_logger.IsInfoEnabled)
{
_logger.Info($"Executed map for '{_index.Name}' index and '{collection}' collection. Got {resultsCount:#,#;;0} map results in {collectionStats.Duration.TotalMilliseconds:#,#;;0} ms.");
}
if (_index.Type.IsMap())
{
_index.SaveLastState();
_indexStorage.WriteLastIndexedEtag(indexContext.Transaction, collection, lastEtag);
}
else
{
_mapReduceContext.ProcessedDocEtags[collection] = lastEtag;
}
}
}
return(moreWorkFound, batchContinuationResult);
}
public override int HandleMap(LazyStringValue lowerId, LazyStringValue id, IEnumerable mapResults, IndexWriteOperation writer, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
throw new NotSupportedException($"Index {Name} is in-memory implementation of a faulty index", _e);
}
public void Cleanup()
{
using (var database = CreateDocumentDatabase())
{
using (var index = AutoMapIndex.CreateNew(1, new AutoMapIndexDefinition("Users", new[] { new IndexField
{
Name = "Name",
Highlighted = false,
Storage = FieldStorage.No
} }), database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "key/1", new DynamicJsonValue
{
["Name"] = "John",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "key/1", null, doc);
}
using (var doc = CreateDocument(context, "key/2", new DynamicJsonValue
{
["Name"] = "Edward",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "key/2", null, doc);
}
using (var doc = CreateDocument(context, "key/3", new DynamicJsonValue
{
["Name"] = "William",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "key/3", null, doc);
}
tx.Commit();
}
var batchStats = new IndexingRunStats();
var stats = new IndexingStatsScope(batchStats);
index.DoIndexingWork(stats, CancellationToken.None);
var tombstones = index.GetLastProcessedDocumentTombstonesPerCollection();
Assert.Equal(1, tombstones.Count);
Assert.Equal(0, tombstones["Users"]);
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Users", 0, 0, 128).Count();
Assert.Equal(0, count);
}
database.DocumentTombstoneCleaner.ExecuteCleanup(null);
using (var tx = context.OpenWriteTransaction())
{
database.DocumentsStorage.Delete(context, "key/1", null);
tx.Commit();
}
tombstones = index.GetLastProcessedDocumentTombstonesPerCollection();
Assert.Equal(1, tombstones.Count);
Assert.Equal(0, tombstones["Users"]);
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Users", 0, 0, 128).Count();
Assert.Equal(1, count);
}
database.DocumentTombstoneCleaner.ExecuteCleanup(null);
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Users", 0, 0, 128).Count();
Assert.Equal(1, count);
}
batchStats = new IndexingRunStats();
stats = new IndexingStatsScope(batchStats);
index.DoIndexingWork(stats, CancellationToken.None);
tombstones = index.GetLastProcessedDocumentTombstonesPerCollection();
Assert.Equal(1, tombstones.Count);
Assert.Equal(4, tombstones["Users"]);
using (var tx = context.OpenWriteTransaction())
{
database.DocumentsStorage.Delete(context, "key/2", null);
tx.Commit();
}
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Users", 0, 0, 128).Count();
Assert.Equal(2, count);
}
database.DocumentTombstoneCleaner.ExecuteCleanup(null);
using (context.OpenReadTransaction())
{
var list = database.DocumentsStorage.GetTombstonesFrom(context, "Users", 0, 0, 128).ToList();
Assert.Equal(1, list.Count);
Assert.Equal(5, list[0].Etag);
}
}
}
}
}
public override IIndexedItemEnumerator GetMapEnumerator(IEnumerable <IndexItem> items, string collection, TransactionOperationContext indexContext, IndexingStatsScope stats, IndexType type)
{
return(new StaticIndexItemEnumerator <DynamicBlittableJson>(items, filter: null, _compiled.Maps[collection], collection, stats, type));
}
public async Task Static_map_reduce_index_with_multiple_outputs_per_document()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition()
{
Name = "Users_ByCount_GroupByLocation",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
Fields =
{
{ "Product", new IndexFieldOptions {
Storage = FieldStorage.Yes
} }
}
}, database))
{
DocumentQueryResult queryResult;
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "orders/1", new DynamicJsonValue
{
["Lines"] = new DynamicJsonArray
{
new DynamicJsonValue
{
["Product"] = "Milk",
["Price"] = 10.5
},
new DynamicJsonValue
{
["Product"] = "Bread",
["Price"] = 10.7
}
},
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Orders"
}
}))
{
database.DocumentsStorage.Put(context, "orders/1", null, doc);
}
using (var doc = CreateDocument(context, "orders/2", new DynamicJsonValue
{
["Lines"] = new DynamicJsonArray
{
new DynamicJsonValue
{
["Product"] = "Milk",
["Price"] = 10.5
}
},
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Orders"
}
}))
{
database.DocumentsStorage.Put(context, "orders/2", null, doc);
}
tx.Commit();
}
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
while (index.DoIndexingWork(scope, CancellationToken.None))
{
}
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Product = 'Milk'"), context, OperationCancelToken.None);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal("Milk", queryResult.Results[0].Data["Product"].ToString());
Assert.Equal(2L, queryResult.Results[0].Data["Count"]);
Assert.Equal(21.0, (LazyNumberValue)queryResult.Results[0].Data["Total"]);
}
}
}
}
public void InitializeForEnumeration(IEnumerable items, TransactionOperationContext indexContext, IndexingStatsScope stats)
{
_items = items;
_indexContext = indexContext;
if (_stats == stats)
{
return;
}
_stats = stats;
_createBlittableResultStats = _stats.For(IndexingOperation.Reduce.CreateBlittableJson, start: false);
}
[InlineData(10, 1, 1)] // nested section
public void Getting_trees(int numberOfDocs, int expectedTreeDepth, int expectedPageCount)
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition
{
Etag = 1,
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
foreach (var documentId in new[] { "orders/1", "orderS/1" })
{
IEnumerable <ReduceTree> trees;
using (index.GetReduceTree(new[] { documentId }, out trees))
{
var result = trees.ToList();
Assert.Equal(2, result.Count);
for (int i = 0; i < 2; i++)
{
var tree = result[i];
Assert.Equal(expectedTreeDepth, tree.Depth);
Assert.Equal(numberOfDocs, tree.NumberOfEntries);
Assert.Equal(expectedPageCount, tree.PageCount);
var hasSource = false;
List <ReduceTreePage> pages;
if (tree.Depth > 1)
{
// real tree
Assert.True(tree.Root.Children.Any());
Assert.Null(tree.Root.Entries);
pages = tree.Root.Children;
}
else
{
// nested section
Assert.Null(tree.Root.Children);
Assert.NotNull(tree.Root.Entries);
pages = new List <ReduceTreePage>
{
tree.Root
};
}
Assert.NotNull(tree.Root.AggregationResult);
foreach (var leafPage in pages)
{
Assert.Null(leafPage.Children);
Assert.NotNull(leafPage.AggregationResult);
foreach (var entry in leafPage.Entries)
{
if (string.IsNullOrEmpty(entry.Source) == false)
{
hasSource = true;
}
Assert.NotNull(entry.Data);
}
}
Assert.True(hasSource);
Assert.Equal(numberOfDocs, pages.Sum(x => x.Entries.Count));
}
}
}
}
}
}
}
public bool CanContinueBatch(DocumentsOperationContext documentsContext, TransactionOperationContext indexingContext, IndexingStatsScope stats, long currentEtag, long maxEtag)
{
throw new NotSupportedException();
}
public void Getting_identifiers_of_source_docs()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition
{
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
Etag = 1
}, database))
{
var numberOfDocs = 100;
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
List <string> result;
IEnumerable <string> ids;
using (index.GetIdentifiersOfMappedDocuments(null, 0, 10, out ids))
{
result = ids.ToList();
Assert.Equal(10, result.Count);
Assert.Equal(result.Count, result.Distinct().Count());
}
using (index.GetIdentifiersOfMappedDocuments(null, 9, 1, out ids))
{
Assert.Equal(1, ids.Count());
Assert.Equal(result[9], ids.First());
}
using (index.GetIdentifiersOfMappedDocuments(null, 100, 10, out ids))
{
Assert.Empty(ids);
}
using (index.GetIdentifiersOfMappedDocuments("orders/3", 0, 1024, out ids))
{
result = ids.ToList();
Assert.Equal(11, result.Count);
Assert.Equal("orders/3", result[0]);
for (var i = 0; i < 10; i++)
{
Assert.Equal($"orders/3{i}", result[i + 1]);
}
}
using (index.GetIdentifiersOfMappedDocuments("prod", 0, 100, out ids))
{
Assert.Empty(ids);
}
}
}
}
}
private void HandleTreeReduction(TransactionOperationContext indexContext, IndexingStatsScope stats,
CancellationToken token, MapReduceResultsStore modifiedStore, LowLevelTransaction lowLevelTransaction,
IndexWriteOperation writer, LazyStringValue reduceKeyHash, Table table)
{
EnsureValidTreeReductionStats(stats);
var tree = modifiedStore.Tree;
var branchesToAggregate = new HashSet <long>();
var parentPagesToAggregate = new HashSet <long>();
var page = new TreePage(null, Constants.Storage.PageSize);
foreach (var modifiedPage in modifiedStore.ModifiedPages)
{
token.ThrowIfCancellationRequested();
page.Base = lowLevelTransaction.GetPage(modifiedPage).Pointer;
stats.RecordReduceTreePageModified(page.IsLeaf);
if (page.IsLeaf == false)
{
Debug.Assert(page.IsBranch);
branchesToAggregate.Add(modifiedPage);
continue;
}
var leafPage = page;
var compressed = leafPage.IsCompressed;
if (compressed)
{
stats.RecordCompressedLeafPage();
}
using (compressed ? (DecompressedLeafPage)(leafPage = tree.DecompressPage(leafPage, skipCache: true)) : null)
{
if (leafPage.NumberOfEntries == 0)
{
if (leafPage.PageNumber != tree.State.RootPageNumber)
{
throw new UnexpectedReduceTreePageException(
$"Encountered empty page which isn't a root. Page {leafPage} in '{tree.Name}' tree.");
}
writer.DeleteReduceResult(reduceKeyHash, stats);
var emptyPageNumber = Bits.SwapBytes(leafPage.PageNumber);
using (Slice.External(indexContext.Allocator, (byte *)&emptyPageNumber, sizeof(long), out Slice pageNumSlice))
table.DeleteByKey(pageNumSlice);
continue;
}
var parentPage = tree.GetParentPageOf(leafPage);
stats.RecordReduceAttempts(leafPage.NumberOfEntries);
try
{
using (var result = AggregateLeafPage(leafPage, lowLevelTransaction, indexContext, token))
{
if (parentPage == -1)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
}
else
{
StoreAggregationResult(leafPage.PageNumber, leafPage.NumberOfEntries, table, result);
parentPagesToAggregate.Add(parentPage);
}
_metrics.MapReduceIndexes.ReducedPerSec.Mark(leafPage.NumberOfEntries);
stats.RecordReduceSuccesses(leafPage.NumberOfEntries);
}
}
catch (Exception e)
{
_index.ThrowIfCorruptionException(e);
LogReductionError(e, reduceKeyHash, stats, updateStats: parentPage == -1, page: leafPage);
}
}
}
long tmp = 0;
using (Slice.External(indexContext.Allocator, (byte *)&tmp, sizeof(long), out Slice pageNumberSlice))
{
foreach (var freedPage in modifiedStore.FreedPages)
{
tmp = Bits.SwapBytes(freedPage);
table.DeleteByKey(pageNumberSlice);
}
}
while (parentPagesToAggregate.Count > 0 || branchesToAggregate.Count > 0)
{
token.ThrowIfCancellationRequested();
var branchPages = parentPagesToAggregate;
parentPagesToAggregate = new HashSet <long>();
foreach (var pageNumber in branchPages)
{
page.Base = lowLevelTransaction.GetPage(pageNumber).Pointer;
try
{
if (page.IsBranch == false)
{
throw new UnexpectedReduceTreePageException("Parent page was found that wasn't a branch, error at " + page);
}
stats.RecordReduceAttempts(page.NumberOfEntries);
var parentPage = tree.GetParentPageOf(page);
using (var result = AggregateBranchPage(page, table, indexContext, branchesToAggregate, token))
{
if (parentPage == -1)
{
writer.DeleteReduceResult(reduceKeyHash, stats);
foreach (var output in result.GetOutputs())
{
writer.IndexDocument(reduceKeyHash, output, stats, indexContext);
}
}
else
{
parentPagesToAggregate.Add(parentPage);
StoreAggregationResult(page.PageNumber, page.NumberOfEntries, table, result);
}
_metrics.MapReduceIndexes.ReducedPerSec.Mark(page.NumberOfEntries);
stats.RecordReduceSuccesses(page.NumberOfEntries);
}
}
catch (Exception e)
{
_index.ThrowIfCorruptionException(e);
LogReductionError(e, reduceKeyHash, stats, updateStats: true, page: page);
}
finally
{
branchesToAggregate.Remove(pageNumber);
}
}
if (parentPagesToAggregate.Count == 0 && branchesToAggregate.Count > 0)
{
// we still have unaggregated branches which were modified but their children were not modified (branch page splitting) so we missed them
parentPagesToAggregate.Add(branchesToAggregate.First());
}
}
}
public bool CanContinueBatch(DocumentsOperationContext documentsContext, TransactionOperationContext indexingContext, IndexingStatsScope stats, long currentEtag, long maxEtag, int count)
{
if (stats.Duration >= _configuration.MapTimeout.AsTimeSpan)
{
stats.RecordMapCompletedReason($"Exceeded maximum configured map duration ({_configuration.MapTimeout.AsTimeSpan}). Was {stats.Duration}");
return(false);
}
if (currentEtag >= maxEtag && stats.Duration >= _configuration.MapTimeoutAfterEtagReached.AsTimeSpan)
{
stats.RecordMapCompletedReason($"Reached maximum etag that was seen when batch started ({maxEtag:#,#;;0}) and map duration ({stats.Duration}) exceeded configured limit ({_configuration.MapTimeoutAfterEtagReached.AsTimeSpan})");
return(false);
}
if (ShouldReleaseTransactionBecauseFlushIsWaiting(stats))
{
return(false);
}
if (_index.CanContinueBatch(stats, documentsContext, indexingContext, count) == false)
{
return(false);
}
return(true);
}
