private void HandleStaticIndexChange(string name, IndexDefinition definition)
{
var creationOptions = IndexCreationOptions.Create;
var currentIndex = GetIndex(name);
IndexDefinitionCompareDifferences currentDifferences = IndexDefinitionCompareDifferences.None;
if (currentIndex != null)
{
creationOptions = GetIndexCreationOptions(definition, currentIndex, out currentDifferences);
}
var replacementIndexName = Constants.Documents.Indexing.SideBySideIndexNamePrefix + definition.Name;
if (creationOptions == IndexCreationOptions.Noop)
{
Debug.Assert(currentIndex != null);
var replacementIndex = GetIndex(replacementIndexName);
if (replacementIndex != null)
{
DeleteIndexInternal(replacementIndex);
}
return;
}
if (creationOptions == IndexCreationOptions.UpdateWithoutUpdatingCompiledIndex)
{
Debug.Assert(currentIndex != null);
var replacementIndex = GetIndex(replacementIndexName);
if (replacementIndex != null)
{
DeleteIndexInternal(replacementIndex);
}
if (currentDifferences != IndexDefinitionCompareDifferences.None)
{
UpdateIndex(definition, currentIndex, currentDifferences);
}
return;
}
UpdateStaticIndexLockModeAndPriority(definition, currentIndex, currentDifferences);
if (creationOptions == IndexCreationOptions.Update)
{
Debug.Assert(currentIndex != null);
definition.Name = replacementIndexName;
var replacementIndex = GetIndex(replacementIndexName);
if (replacementIndex != null)
{
creationOptions = GetIndexCreationOptions(definition, replacementIndex, out IndexDefinitionCompareDifferences sideBySideDifferences);
if (creationOptions == IndexCreationOptions.Noop)
{
return;
}
if (creationOptions == IndexCreationOptions.UpdateWithoutUpdatingCompiledIndex)
{
UpdateIndex(definition, replacementIndex, sideBySideDifferences);
return;
}
DeleteIndexInternal(replacementIndex);
}
}
Index index;
switch (definition.Type)
{
case IndexType.Map:
index = MapIndex.CreateNew(definition, _documentDatabase);
break;
case IndexType.MapReduce:
index = MapReduceIndex.CreateNew(definition, _documentDatabase);
break;
default:
throw new NotSupportedException($"Cannot create {definition.Type} index from IndexDefinition");
}
CreateIndexInternal(index);
}
public async Task Reduction_should_ignore_overflow_pages(long numberOfDocs)
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition()
{
Etag = 10,
Name = "Users_ByLocation",
Maps = { @"from user in docs.Users
select new { Location = user.Location, Count = 1 }" },
Reduce = @"from result in results
group result by result.Location into g
select new
{
Location = g.Key,
Count = g.Sum(x=> x.Count)
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
var bytes = new byte[4096];
new Random(2).NextBytes(bytes); // TODO arek - seed
var randomLocation = Encoding.ASCII.GetString(bytes);
using (var tx = context.OpenWriteTransaction())
{
for (int i = 0; i < numberOfDocs; i++)
{
var user = new DynamicJsonValue()
{
["Location"] = randomLocation,
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
};
using (var doc = CreateDocument(context, $"users/{i}", user))
{
database.DocumentsStorage.Put(context, $"users/{i}", null, doc);
}
}
tx.Commit();
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
Assert.Equal(numberOfDocs, firstRunStats.MapAttempts);
Assert.Equal(numberOfDocs, firstRunStats.MapSuccesses);
Assert.Equal(0, firstRunStats.MapErrors);
var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.False(queryResult.IsStale);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal(numberOfDocs, queryResult.Results[0].Data["Count"]);
}
}
}
}
public void PageModificationInAnyTreeMustRemoveItFromListOfFreedPagesInAllStores()
{
using (var database = CreateDocumentDatabase())
using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
{
Name = "Users_ByCount_GroupByLocation",
Maps = { "from user in docs.Users select new { user.Location, Count = 1 }" },
Reduce =
"from result in results group result by result.Location into g select new { Location = g.Key, Count = g.Sum(x => x.Count) }",
Type = IndexType.MapReduce,
Fields =
{
{ "Location", new IndexFieldOptions {
Storage = FieldStorage.Yes
} },
{ "Count", new IndexFieldOptions {
Storage = FieldStorage.Yes
} }
}
}, database))
{
index._threadAllocations = NativeMemory.CurrentThreadStats;
var mapReduceContext = new MapReduceIndexingContext();
using (var contextPool = new TransactionContextPool(database.DocumentsStorage.Environment))
{
var indexStorage = new IndexStorage(index, contextPool, database);
var reducer = new ReduceMapResultsOfStaticIndex(index, index._compiled.Reduce, index.Definition, indexStorage, new MetricCounters(), mapReduceContext);
using (index._contextPool.AllocateOperationContext(out TransactionOperationContext indexContext))
{
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store1 = new MapReduceResultsStore(1, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
var store2 = new MapReduceResultsStore(2, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
mapReduceContext.StoreByReduceKeyHash.Add(1, store1);
mapReduceContext.StoreByReduceKeyHash.Add(2, store2);
// we're cheating here a bit as the originally this issue was reproduced on very large amount of data
//
// we choose page 542 because it's going to be used when calling store1.Add() below
// let's pretend this page that was freed as the result of deletion in store2
// the important thing is that store1 will be processed before processing store2 and we'll store the aggregation result for page 542 in PageNumberToReduceResult table
// the issue was that modification of page 542 in the tree of store1 didn't remove it from FreedPages of store2
// in result the processing of store2 removed page 542 from the table
long pageNumber = 543;
if (tx.InnerTransaction.LowLevelTransaction.Environment.Options.ForceUsing32BitsPager || PlatformDetails.Is32Bits)
{
// in 32 bits we might allocate different pages, 94 is going to be used during store1.Add() calls
pageNumber = 95;
}
mapReduceContext.FreedPages.Add(pageNumber);
for (int i = 0; i < 200; i++)
{
using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
{
["Count"] = 1,
["Location"] = new string('c', 1024)
}, $"users/{i}"))
{
store1.Add(i, mappedResult);
}
}
var writeOperation = new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, null));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
Assert.DoesNotContain(pageNumber, mapReduceContext.FreedPages);
var table = indexContext.Transaction.InnerTransaction.OpenTable(ReduceMapResultsBase <MapReduceIndexDefinition> .ReduceResultsSchema,
ReduceMapResultsBase <MapReduceIndexDefinition> .PageNumberToReduceResultTableName);
var page = Bits.SwapBytes(pageNumber);
unsafe
{
using (Slice.External(indexContext.Allocator, (byte *)&page, sizeof(long), out Slice pageSlice))
{
Assert.True(table.ReadByKey(pageSlice, out TableValueReader tvr));
}
}
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
}
}
}
}
}
public void When_map_results_do_not_change_then_we_skip_the_reduce_phase(int numberOfDocs)
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(1, new IndexDefinition()
{
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
Assert.Equal(numberOfDocs, firstRunStats.MapAttempts);
Assert.Equal(numberOfDocs, firstRunStats.MapSuccesses);
Assert.Equal(0, firstRunStats.MapErrors);
Assert.True(firstRunStats.ReduceAttempts > 0);
Assert.True(firstRunStats.ReduceSuccesses > 0);
Assert.Equal(0, firstRunStats.ReduceErrors);
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
order["RefNumber"] = "456";
PutOrder(database, order, context, i);
}
var secondRunStats = new IndexingRunStats();
scope = new IndexingStatsScope(secondRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
Assert.Equal(firstRunStats.MapAttempts, secondRunStats.MapAttempts);
Assert.Equal(firstRunStats.MapSuccesses, secondRunStats.MapSuccesses);
Assert.Equal(0, secondRunStats.MapErrors);
Assert.Equal(0, secondRunStats.ReduceAttempts);
Assert.Equal(0, secondRunStats.ReduceSuccesses);
Assert.Equal(0, secondRunStats.ReduceErrors);
}
}
}
}
private static void when_there_are_multiple_map_results_for_multiple_indexes(ITransactionalStorage transactionalStorage)
{
transactionalStorage.Initialize(new DummyUuidGenerator());
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a", true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.True(actionsAccessor.Staleness.IsReduceStale("a"));
Assert.True(actionsAccessor.Staleness.IsReduceStale("b"));
Assert.True(actionsAccessor.Staleness.IsReduceStale("c"));
});
}
public override void Execute(WorkContext context)
{
if (ReduceKeys.Length == 0)
{
return;
}
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(Index);
if (viewGenerator == null)
{
return; // deleted view?
}
context.TransactionaStorage.Batch(actions =>
{
var itemsToFind = ReduceKeys
.Select(reduceKey => new GetMappedResultsParams(Index, reduceKey, MapReduceIndex.ComputeHash(Index, reduceKey)))
.OrderBy(x => x.ViewAndReduceKeyHashed, new ByteComparer())
.ToArray();
var mappedResults = actions.MappedResults.GetMappedResults(itemsToFind)
.Select(JsonToExpando.Convert);
var sp = Stopwatch.StartNew();
log.DebugFormat("Starting to read {0} reduce keys for index {1}", ReduceKeys.Length, Index);
var results = mappedResults.ToArray();
log.DebugFormat("Read {0} reduce keys in {1} with {2} results for index {3}", ReduceKeys.Length, sp.Elapsed, results.Length, Index);
sp = Stopwatch.StartNew();
context.IndexStorage.Reduce(Index, viewGenerator, results, context, actions, ReduceKeys);
log.DebugFormat("Indexed {0} reduce keys in {1} with {2} results for index {3}", ReduceKeys.Length, sp.Elapsed,
results.Length, Index);
});
}
public static unsafe long CalculateIndexEtag(MapReduceIndex index, int length, byte *indexEtagBytes, byte *writePos, DocumentsOperationContext documentsContext, TransactionOperationContext indexContext)
{
return(CalculateIndexEtag(index, index._compiled, length, indexEtagBytes, writePos, documentsContext, indexContext));
}
public void CanUpdateValue()
{
transactionalStorage.Batch(actions => actions.MappedResults.PutMappedResult("CommentCountsByBlog", "123", "1", JObject.Parse("{'a': 'abc'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1")));
transactionalStorage.Batch(actions => actions.MappedResults.PutMappedResult("CommentCountsByBlog", "123", "1", JObject.Parse("{'a': 'def'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1")));
}
public void CanStoreAndGetValues()
{
transactionalStorage.Batch(actions =>
{
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "123", "1", JObject.Parse("{'a': 'abc'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"));
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "324", "2", JObject.Parse("{'a': 'def'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "2"));
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "321", "1", JObject.Parse("{'a': 'ijg'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"));
});
transactionalStorage.Batch(actions =>
{
var vals = actions.MappedResults.GetMappedResults("CommentCountsByBlog", "1", MapReduceIndex.ComputeHash("CommentCountsByBlog", "1")).ToArray();
Assert.Equal(2, vals.Length);
Assert.Contains("abc", vals[0].ToString());
Assert.Contains("ijg", vals[1].ToString());
});
}
[InlineData(10)] // nested section
public void Getting_trees_for_multiple_docs(int numberOfDocs)
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition
{
Etag = 1,
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
var docIds = Enumerable.Range(0, numberOfDocs).Select(x => x % 2 == 0 ? "orders/" + x : "Orders/" + x).ToArray();
IEnumerable <ReduceTree> trees;
using (index.GetReduceTree(docIds, out trees))
{
var result = trees.ToList();
Assert.Equal(2, result.Count);
for (int i = 0; i < 2; i++)
{
var tree = result[0];
List <ReduceTreePage> pages;
if (tree.Depth > 1)
{
// real tree
pages = tree.Root.Children;
}
else
{
// nested section
pages = new List <ReduceTreePage>
{
tree.Root
};
}
Assert.NotNull(tree.Root.AggregationResult);
var seenSources = new HashSet <string>();
foreach (var leafPage in pages)
{
foreach (var entry in leafPage.Entries)
{
Assert.NotNull(entry.Source);
seenSources.Add(entry.Source);
}
}
Assert.Equal(numberOfDocs, seenSources.Count);
Assert.Equal(numberOfDocs, pages.Sum(x => x.Entries.Count));
}
}
}
}
}
}
public void CanStoreValues()
{
transactionalStorage.Batch(actions =>
{
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "123", "1", JObject.Parse("{'a': 'abc'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"));
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "324", "2", JObject.Parse("{'a': 'def'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "2"));
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "321", "1", JObject.Parse("{'a': 'ijg'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"));
});
}
public void Getting_identifiers_of_source_docs()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition
{
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
Etag = 1
}, database))
{
var numberOfDocs = 100;
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
List <string> result;
IEnumerable <string> ids;
using (index.GetIdentifiersOfMappedDocuments(null, 0, 10, out ids))
{
result = ids.ToList();
Assert.Equal(10, result.Count);
Assert.Equal(result.Count, result.Distinct().Count());
}
using (index.GetIdentifiersOfMappedDocuments(null, 9, 1, out ids))
{
Assert.Equal(1, ids.Count());
Assert.Equal(result[9], ids.First());
}
using (index.GetIdentifiersOfMappedDocuments(null, 100, 10, out ids))
{
Assert.Empty(ids);
}
using (index.GetIdentifiersOfMappedDocuments("orders/3", 0, 1024, out ids))
{
result = ids.ToList();
Assert.Equal(11, result.Count);
Assert.Equal("orders/3", result[0]);
for (var i = 0; i < 10; i++)
{
Assert.Equal($"orders/3{i}", result[i + 1]);
}
}
using (index.GetIdentifiersOfMappedDocuments("prod", 0, 100, out ids))
{
Assert.Empty(ids);
}
}
}
}
}
[InlineData(10, 1, 1)] // nested section
public void Getting_trees(int numberOfDocs, int expectedTreeDepth, int expectedPageCount)
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition
{
Etag = 1,
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
foreach (var documentId in new[] { "orders/1", "orderS/1" })
{
IEnumerable <ReduceTree> trees;
using (index.GetReduceTree(new[] { documentId }, out trees))
{
var result = trees.ToList();
Assert.Equal(2, result.Count);
for (int i = 0; i < 2; i++)
{
var tree = result[i];
Assert.Equal(expectedTreeDepth, tree.Depth);
Assert.Equal(numberOfDocs, tree.NumberOfEntries);
Assert.Equal(expectedPageCount, tree.PageCount);
var hasSource = false;
List <ReduceTreePage> pages;
if (tree.Depth > 1)
{
// real tree
Assert.True(tree.Root.Children.Any());
Assert.Null(tree.Root.Entries);
pages = tree.Root.Children;
}
else
{
// nested section
Assert.Null(tree.Root.Children);
Assert.NotNull(tree.Root.Entries);
pages = new List <ReduceTreePage>
{
tree.Root
};
}
Assert.NotNull(tree.Root.AggregationResult);
foreach (var leafPage in pages)
{
Assert.Null(leafPage.Children);
Assert.NotNull(leafPage.AggregationResult);
foreach (var entry in leafPage.Entries)
{
if (string.IsNullOrEmpty(entry.Source) == false)
{
hasSource = true;
}
Assert.NotNull(entry.Data);
}
}
Assert.True(hasSource);
Assert.Equal(numberOfDocs, pages.Sum(x => x.Entries.Count));
}
}
}
}
}
}
}
public void CanDeleteValueByView()
{
transactionalStorage.Batch(actions =>
{
actions.PutMappedResult("CommentCountsByBlog1", "123", "1", "{'a': 'abc'}", MapReduceIndex.ComputeHash("CommentCountsByBlog1", "1"));
actions.PutMappedResult("CommentCountsByBlog2", "123", "1", "{'a': 'abc'}", MapReduceIndex.ComputeHash("CommentCountsByBlog2", "1"));
});
transactionalStorage.Batch(actions =>
{
actions.DeleteMappedResultsForView("CommentCountsByBlog2");
});
transactionalStorage.Batch(actions =>
{
Assert.NotEmpty(actions.GetMappedResults("CommentCountsByBlog1", "1", MapReduceIndex.ComputeHash("CommentCountsByBlog1", "1")));
Assert.Empty(actions.GetMappedResults("CommentCountsByBlog2", "1", MapReduceIndex.ComputeHash("CommentCountsByBlog2", "1")));
});
}
public OutputReduceToCollectionCommand(DocumentDatabase database, string outputReduceToCollection, MapReduceIndex index)
{
_database = database;
_outputReduceToCollection = outputReduceToCollection;
_index = index;
_jsonContext = JsonOperationContext.ShortTermSingleUse();
}
public void CanUpdateValueAndGetUpdatedValues()
{
transactionalStorage.Batch(actions =>
{
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "123", "1", JObject.Parse("{'a': 'abc'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"));
});
transactionalStorage.Batch(actions =>
{
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "123", "1", JObject.Parse("{'a': 'def'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"));
});
transactionalStorage.Batch(actions =>
{
var strings = actions.MappedResults.GetMappedResults("CommentCountsByBlog", "1", MapReduceIndex.ComputeHash("CommentCountsByBlog", "1")).Select(x => x.ToString()).ToArray();
Assert.Contains("def", strings[0]);
});
}
public OutputReduceToCollectionCommand(DocumentDatabase database, string outputReduceToCollection, MapReduceIndex index, JsonOperationContext context)
: this(database, outputReduceToCollection)
{
_index = index;
_indexContext = context;
}
public void CanDeleteValueByDocumentId()
{
transactionalStorage.Batch(actions =>
{
actions.MappedResults.PutMappedResult("CommentCountsByBlog1", "123", "1", JObject.Parse("{'a': 'abc'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog1", "1"));
actions.MappedResults.PutMappedResult("CommentCountsByBlog2", "123", "1", JObject.Parse("{'a': 'abc'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog2", "1"));
});
transactionalStorage.Batch(actions =>
{
actions.MappedResults.DeleteMappedResultsForDocumentId("123", "CommentCountsByBlog2");
actions.MappedResults.DeleteMappedResultsForDocumentId("123", "CommentCountsByBlog1");
});
transactionalStorage.Batch(actions =>
{
Assert.Empty(actions.MappedResults.GetMappedResults("CommentCountsByBlog1", "1", MapReduceIndex.ComputeHash("CommentCountsByBlog1", "1")));
Assert.Empty(actions.MappedResults.GetMappedResults("CommentCountsByBlog2", "1", MapReduceIndex.ComputeHash("CommentCountsByBlog2", "1")));
});
}
public int CreateIndex(IndexDefinition definition)
{
if (definition == null)
{
throw new ArgumentNullException(nameof(definition));
}
lock (_locker)
{
Index existingIndex;
var lockMode = ValidateIndexDefinition(definition.Name, out existingIndex);
if (lockMode == IndexLockMode.LockedIgnore)
{
return(existingIndex.IndexId);
}
definition.RemoveDefaultValues();
switch (GetIndexCreationOptions(definition, existingIndex))
{
case IndexCreationOptions.Noop:
return(existingIndex.IndexId);
case IndexCreationOptions.UpdateWithoutUpdatingCompiledIndex:
switch (definition.Type)
{
case IndexType.Map:
MapIndex.Update(existingIndex, definition, _documentDatabase);
break;
case IndexType.MapReduce:
MapReduceIndex.Update(existingIndex, definition, _documentDatabase);
break;
default:
throw new NotSupportedException($"Cannot update {definition.Type} index from IndexDefinition");
}
return(existingIndex.IndexId);
case IndexCreationOptions.Update:
DeleteIndex(existingIndex.IndexId);
break;
}
var indexId = _indexes.GetNextIndexId();
Index index;
switch (definition.Type)
{
case IndexType.Map:
index = MapIndex.CreateNew(indexId, definition, _documentDatabase);
break;
case IndexType.MapReduce:
index = MapReduceIndex.CreateNew(indexId, definition, _documentDatabase);
break;
default:
throw new NotSupportedException($"Cannot create {definition.Type} index from IndexDefinition");
}
return(CreateIndexInternal(index, indexId));
}
}
public JintLuceneDocumentConverter(MapReduceIndex index, bool storeValue = false)
: base(index, index.Definition.IndexDefinition, storeValue: storeValue)
{
}
public async Task By_complex_object_and_array()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(1, new IndexDefinition()
{
Name = "Users_GroupByLocationAndResidenceAddress",
Maps = { @"from user in docs.Users select new {
user.Hobbies,
user.ResidenceAddress,
Count = 1
}" },
Reduce = @"from result in results group result by new { result.Hobbies, result.ResidenceAddress } into g select new {
g.Key.Hobbies,
g.Key.ResidenceAddress,
Count = g.Sum(x => x.Count)
}",
Fields = new Dictionary <string, IndexFieldOptions>()
{
{ "Hobbies", new IndexFieldOptions()
{
Indexing = FieldIndexing.Analyzed,
} },
{ "ResidenceAddress", new IndexFieldOptions()
{
Indexing = FieldIndexing.Analyzed,
} }
}
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
Put_docs(context, database);
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
while (index.DoIndexingWork(scope, CancellationToken.None))
{
}
var queryResult = await index.Query(new IndexQueryServerSide(), context, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
context.ResetAndRenew();
queryResult = await index.Query(new IndexQueryServerSide()
{
Query = @"Hobbies:music"
}, context, OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(1, results.Count);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal("music", ((BlittableJsonReaderArray)results[0].Data["Hobbies"])[0].ToString());
Assert.Equal("sport", ((BlittableJsonReaderArray)results[0].Data["Hobbies"])[1].ToString());
Assert.Equal(@"{""Country"":""UK""}", results[0].Data["ResidenceAddress"].ToString());
Assert.Equal(2L, results[0].Data["Count"]);
}
}
}
}
public static bool IsStale(MapReduceIndex index, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, long?cutoff)
{
return(IsStale(index, index._compiled, databaseContext, indexContext, cutoff));
}
private static void when_there_are_updates_to_map_reduce_results(ITransactionalStorage transactionalStorage)
{
var dummyUuidGenerator = new DummyUuidGenerator();
transactionalStorage.Initialize(dummyUuidGenerator);
Guid a = Guid.Empty;
Guid b = Guid.Empty;
Guid c = Guid.Empty;
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a", true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
a = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
b = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
c = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("a", a, false, 100).Count());
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("b", b, false, 100).Count());
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("c", c, false, 100).Count());
});
}
public async Task Static_map_reduce_index_with_multiple_outputs_per_document()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
{
Name = "Users_ByCount_GroupByLocation",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
Fields =
{
{ "Product", new IndexFieldOptions {
Storage = FieldStorage.Yes
} }
}
}, database))
{
DocumentQueryResult queryResult;
using (var queryContext = QueryOperationContext.ShortTermSingleUse(database))
{
var context = queryContext.Documents;
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "orders/1", new DynamicJsonValue
{
["Lines"] = new DynamicJsonArray
{
new DynamicJsonValue
{
["Product"] = "Milk",
["Price"] = 10.5
},
new DynamicJsonValue
{
["Product"] = "Bread",
["Price"] = 10.7
}
},
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Orders"
}
}))
{
database.DocumentsStorage.Put(context, "orders/1", null, doc);
}
using (var doc = CreateDocument(context, "orders/2", new DynamicJsonValue
{
["Lines"] = new DynamicJsonArray
{
new DynamicJsonValue
{
["Product"] = "Milk",
["Price"] = 10.5
}
},
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Orders"
}
}))
{
database.DocumentsStorage.Put(context, "orders/2", null, doc);
}
tx.Commit();
}
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
while (index.DoIndexingWork(scope, CancellationToken.None))
{
}
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), queryContext, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
}
using (var context = QueryOperationContext.ShortTermSingleUse(database))
{
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Product = 'Milk'"), context, OperationCancelToken.None);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal("Milk", queryResult.Results[0].Data["Product"].ToString());
Assert.Equal(2L, queryResult.Results[0].Data["Count"]);
Assert.Equal(21.0, (LazyNumberValue)queryResult.Results[0].Data["Total"]);
}
}
}
}
private static void when_there_are_multiple_map_results_and_we_ask_for_results_will_get_latest(ITransactionalStorage transactionalStorage)
{
transactionalStorage.Initialize(new DummyUuidGenerator());
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a", true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("a", Guid.Empty, false, 100).Count());
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("b", Guid.Empty, false, 100).Count());
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("c", Guid.Empty, false, 100).Count());
});
}
public async Task By_multiple_complex_objects()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
{
Name = "Users_GroupByLocationAndResidenceAddress",
Maps = { @"from user in docs.Users select new {
user.Location,
user.ResidenceAddress,
Count = 1
}" },
Reduce = @"from result in results group result by new { result.Location, result.ResidenceAddress } into g select new {
g.Key.Location,
g.Key.ResidenceAddress,
Count = g.Sum(x => x.Count)
}",
Fields = new Dictionary <string, IndexFieldOptions>()
{
{ "Location", new IndexFieldOptions()
{
Indexing = FieldIndexing.Search,
} },
{ "ResidenceAddress", new IndexFieldOptions()
{
Indexing = FieldIndexing.Search,
} }
}
}, database))
{
DocumentQueryResult queryResult;
using (var context = QueryOperationContext.ShortTermSingleUse(database))
{
Put_docs(context.Documents, database);
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
while (index.DoIndexingWork(scope, CancellationToken.None))
{
}
queryResult =
await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
}
using (var context = QueryOperationContext.ShortTermSingleUse(database))
{
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Location = 'Poland'"), context, OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(1, results.Count);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal(@"{""Country"":""Poland"",""State"":""Pomerania""}", results[0].Data["Location"].ToString());
Assert.Equal(@"{""Country"":""UK""}", results[0].Data["ResidenceAddress"].ToString());
Assert.Equal(2L, results[0].Data["Count"]);
}
}
}
}
public async Task The_simpliest_static_map_reduce_index()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition()
{
Name = "Users_ByCount_GroupByLocation",
Maps = { @"from user in docs.Users select new {
user.Location,
CountInteger = 1,
CountDouble = 1.0,
CastedInteger = 1
}" },
Reduce = @"from result in results group result by result.Location into g select new {
Location = g.Key,
CountInteger = g.Sum(x => x.CountInteger),
CountDouble = g.Sum(x => x.CountDouble),
CastedInteger = g.Sum(x => (int)x.CastedInteger)
}"
}, database))
{
DocumentQueryResult queryResult;
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
{
["Location"] = "Poland",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "users/1", null, doc);
}
using (var doc = CreateDocument(context, "users/2", new DynamicJsonValue
{
["Location"] = "Poland",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "users/2", null, doc);
}
tx.Commit();
}
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
index.DoIndexingWork(scope, CancellationToken.None);
Assert.Equal(2, batchStats.MapAttempts);
Assert.Equal(2, batchStats.MapSuccesses);
Assert.Equal(0, batchStats.MapErrors);
Assert.Equal(2, batchStats.ReduceAttempts);
Assert.Equal(2, batchStats.ReduceSuccesses);
Assert.Equal(0, batchStats.ReduceErrors);
queryResult =
await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.Equal(1, queryResult.Results.Count);
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Location = 'Poland'"), context, OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(1, results.Count);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal("Poland", results[0].Data["Location"].ToString());
Assert.Equal(2L, results[0].Data["CountInteger"]);
Assert.Equal(2.0, (LazyNumberValue)results[0].Data["CountDouble"]);
Assert.Equal(2L, results[0].Data["CastedInteger"]);
}
}
}
}
public static bool CanReplace(MapReduceIndex index, bool isStale, DocumentDatabase database, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext)
{
return(isStale == false);
}
public static bool IsStaleDueToReferences(MapReduceIndex index, DocumentsOperationContext databaseContext, TransactionOperationContext indexContext, long?referenceCutoff, List <string> stalenessReasons)
{
return(IsStaleDueToReferences(index, index._compiled, databaseContext, indexContext, referenceCutoff, stalenessReasons));
}
public void CanAddmultipleValuesForTheSameKey()
{
transactionalStorage.Batch(actions =>
{
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "123", "1", RavenJObject.Parse("{'a': 'abc'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"));
});
transactionalStorage.Batch(actions =>
{
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "123", "1", RavenJObject.Parse("{'a': 'def'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"));
});
transactionalStorage.Batch(actions =>
{
var strings = actions.MappedResults.GetMappedResults(new GetMappedResultsParams("CommentCountsByBlog", "1", MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"))).Select(x => x.ToString()).ToArray();
Assert.Equal(2, strings.Length);
Assert.Contains("abc", strings[0]);
Assert.Contains("def", strings[1]);
});
}
