public async Task By_single_array_object()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition()
{
Etag = 10,
Name = "Users_GroupByHobbies",
Maps = { @"from user in docs.Users select new {
user.Hobbies,
Count = 1
}" },
Reduce = @"from result in results group result by result.Hobbies into g select new {
Hobbies = g.Key,
Count = g.Sum(x => x.Count)
}",
Fields = new Dictionary <string, IndexFieldOptions>()
{
{ "Hobbies", new IndexFieldOptions()
{
Indexing = FieldIndexing.Search,
} }
}
}, database))
{
DocumentQueryResult queryResult;
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
Put_docs(context, database);
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
index.DoIndexingWork(scope, CancellationToken.None);
queryResult =
await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Hobbies = 'music'"), context, OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(1, results.Count);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal("music", ((BlittableJsonReaderArray)results[0].Data["Hobbies"])[0].ToString());
Assert.Equal("sport", ((BlittableJsonReaderArray)results[0].Data["Hobbies"])[1].ToString());
Assert.Equal(2L, results[0].Data["Count"]);
foreach (var document in results)
{
document.Data.Dispose();
}
}
}
}
}
public async Task By_single_complex_object()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(1, new IndexDefinition()
{
Name = "Users_ByCount_GroupByLocation",
Maps = { @"from user in docs.Users select new {
user.Location,
Count = 1
}" },
Reduce = @"from result in results group result by result.Location into g select new {
Location = g.Key,
Count = g.Sum(x => x.Count)
}",
Fields = new Dictionary <string, IndexFieldOptions>()
{
{ "Location", new IndexFieldOptions()
{
Indexing = FieldIndexing.Analyzed,
} }
}
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
Put_docs(context, database);
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
index.DoIndexingWork(scope, CancellationToken.None);
var queryResult = await index.Query(new IndexQueryServerSide(), context, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
context.ResetAndRenew();
queryResult = await index.Query(new IndexQueryServerSide()
{
Query = @"Location:Poland"
}, context, OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(1, results.Count);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal(@"{""Country"":""Poland"",""State"":""Pomerania""}", results[0].Data["Location"].ToString());
Assert.Equal(2L, results[0].Data["Count"]);
}
}
}
}
public void ErrorShouldIncludeTheActualItemAndReduceKey()
{
var numberOfDocs = 100;
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
{
Name = "Users_DivideByZero",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, FakeValue = 0 }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
FakeValue = (long) (128 / g.Sum(x=> x.Total) - g.Sum(x=> x.Count))
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var stats = new IndexingRunStats();
var scope = new IndexingStatsScope(stats);
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
try
{
index.DoIndexingWork(scope, cts.Token);
}
catch (ExcessiveNumberOfReduceErrorsException)
{
// expected
}
List <IndexingError> indexingErrors = stats.Errors;
Assert.Equal(1, indexingErrors.Count);
Assert.Contains(@"current item to reduce: {""Product"":""Milk"",""FakeValue"":0}", indexingErrors.First().Error);
Assert.Equal(@"Reduce key: { 'Product' : Milk }", indexingErrors.First().Document);
}
}
}
}
public async Task MultipleReduceKeys(int numberOfUsers, string[] locations)
{
using (var db = CreateDocumentDatabase())
using (var index = AutoMapReduceIndex.CreateNew(GetUsersCountByLocationIndexDefinition(), db))
{
CreateUsers(db, numberOfUsers, locations);
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
while (index.DoIndexingWork(scope, cts.Token))
{
;
}
Assert.Equal(numberOfUsers, batchStats.MapAttempts);
Assert.Equal(numberOfUsers, batchStats.MapSuccesses);
Assert.Equal(0, batchStats.MapErrors);
Assert.True(batchStats.ReduceAttempts >= numberOfUsers, $"{batchStats.ReduceAttempts} >= {numberOfUsers}");
Assert.True(batchStats.ReduceSuccesses >= numberOfUsers, $"{batchStats.ReduceSuccesses} >= {numberOfUsers}");
Assert.Equal(batchStats.ReduceAttempts, batchStats.ReduceSuccesses);
Assert.Equal(0, batchStats.ReduceErrors);
using (var context = DocumentsOperationContext.ShortTermSingleUse(db))
{
var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'")
{
WaitForNonStaleResultsTimeout = TimeSpan.FromMinutes(1)
}, context, OperationCancelToken.None);
Assert.False(queryResult.IsStale);
var results = queryResult.Results;
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected, results[i].Data["Count"]);
}
}
}
}
public async Task The_simpliest_static_map_reduce_index()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition()
{
Name = "Users_ByCount_GroupByLocation",
Maps = { @"from user in docs.Users select new {
user.Location,
CountInteger = 1,
CountDouble = 1.0,
CastedInteger = 1
}" },
Reduce = @"from result in results group result by result.Location into g select new {
Location = g.Key,
CountInteger = g.Sum(x => x.CountInteger),
CountDouble = g.Sum(x => x.CountDouble),
CastedInteger = g.Sum(x => (int)x.CastedInteger)
}"
}, database))
{
DocumentQueryResult queryResult;
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
{
["Location"] = "Poland",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "users/1", null, doc);
}
using (var doc = CreateDocument(context, "users/2", new DynamicJsonValue
{
["Location"] = "Poland",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "users/2", null, doc);
}
tx.Commit();
}
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
index.DoIndexingWork(scope, CancellationToken.None);
Assert.Equal(2, batchStats.MapAttempts);
Assert.Equal(2, batchStats.MapSuccesses);
Assert.Equal(0, batchStats.MapErrors);
Assert.Equal(2, batchStats.ReduceAttempts);
Assert.Equal(2, batchStats.ReduceSuccesses);
Assert.Equal(0, batchStats.ReduceErrors);
queryResult =
await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.Equal(1, queryResult.Results.Count);
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Location = 'Poland'"), context, OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(1, results.Count);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal("Poland", results[0].Data["Location"].ToString());
Assert.Equal(2L, results[0].Data["CountInteger"]);
Assert.Equal(2.0, (LazyNumberValue)results[0].Data["CountDouble"]);
Assert.Equal(2L, results[0].Data["CastedInteger"]);
}
}
}
}
public async Task Static_map_reduce_index_with_multiple_outputs_per_document()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition()
{
Name = "Users_ByCount_GroupByLocation",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
Fields =
{
{ "Product", new IndexFieldOptions {
Storage = FieldStorage.Yes
} }
}
}, database))
{
DocumentQueryResult queryResult;
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "orders/1", new DynamicJsonValue
{
["Lines"] = new DynamicJsonArray
{
new DynamicJsonValue
{
["Product"] = "Milk",
["Price"] = 10.5
},
new DynamicJsonValue
{
["Product"] = "Bread",
["Price"] = 10.7
}
},
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Orders"
}
}))
{
database.DocumentsStorage.Put(context, "orders/1", null, doc);
}
using (var doc = CreateDocument(context, "orders/2", new DynamicJsonValue
{
["Lines"] = new DynamicJsonArray
{
new DynamicJsonValue
{
["Product"] = "Milk",
["Price"] = 10.5
}
},
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Orders"
}
}))
{
database.DocumentsStorage.Put(context, "orders/2", null, doc);
}
tx.Commit();
}
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
while (index.DoIndexingWork(scope, CancellationToken.None))
{
}
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Product = 'Milk'"), context, OperationCancelToken.None);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal("Milk", queryResult.Results[0].Data["Product"].ToString());
Assert.Equal(2L, queryResult.Results[0].Data["Count"]);
Assert.Equal(21.0, (LazyNumberValue)queryResult.Results[0].Data["Total"]);
}
}
}
}
[InlineData(10)] // nested section
public void Getting_trees_for_multiple_docs(int numberOfDocs)
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition
{
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
var docIds = Enumerable.Range(0, numberOfDocs).Select(x => x % 2 == 0 ? "orders/" + x : "Orders/" + x).ToArray();
IEnumerable <ReduceTree> trees;
using (index.GetReduceTree(docIds, out trees))
{
var result = trees.ToList();
Assert.Equal(2, result.Count);
for (int i = 0; i < 2; i++)
{
var tree = result[0];
List <ReduceTreePage> pages;
if (tree.Depth > 1)
{
// real tree
pages = tree.Root.Children;
}
else
{
// nested section
pages = new List <ReduceTreePage>
{
tree.Root
};
}
Assert.NotNull(tree.Root.AggregationResult);
var seenSources = new HashSet <string>();
foreach (var leafPage in pages)
{
foreach (var entry in leafPage.Entries)
{
Assert.NotNull(entry.Source);
seenSources.Add(entry.Source);
}
}
Assert.Equal(numberOfDocs, seenSources.Count);
Assert.Equal(numberOfDocs, pages.Sum(x => x.Entries.Count));
}
}
}
}
}
}
public void Getting_identifiers_of_source_docs()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition
{
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}"
}, database))
{
var numberOfDocs = 100;
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
List <string> result;
IEnumerable <string> ids;
using (index.GetIdentifiersOfMappedDocuments(null, 0, 10, out ids))
{
result = ids.ToList();
Assert.Equal(10, result.Count);
Assert.Equal(result.Count, result.Distinct().Count());
}
using (index.GetIdentifiersOfMappedDocuments(null, 9, 1, out ids))
{
Assert.Equal(1, ids.Count());
Assert.Equal(result[9], ids.First());
}
using (index.GetIdentifiersOfMappedDocuments(null, 100, 10, out ids))
{
Assert.Empty(ids);
}
using (index.GetIdentifiersOfMappedDocuments("orders/3", 0, 1024, out ids))
{
result = ids.ToList();
Assert.Equal(11, result.Count);
Assert.Equal("orders/3", result[0]);
for (var i = 0; i < 10; i++)
{
Assert.Equal($"orders/3{i}", result[i + 1]);
}
}
using (index.GetIdentifiersOfMappedDocuments("prod", 0, 100, out ids))
{
Assert.Empty(ids);
}
}
}
}
}
[InlineData(10, 1, 1)] // nested section
public void Getting_trees(int numberOfDocs, int expectedTreeDepth, int expectedPageCount)
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition
{
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
foreach (var documentId in new[] { "orders/1", "orderS/1" })
{
IEnumerable <ReduceTree> trees;
using (index.GetReduceTree(new[] { documentId }, out trees))
{
var result = trees.ToList();
Assert.Equal(2, result.Count);
for (int i = 0; i < 2; i++)
{
var tree = result[i];
Assert.Equal(expectedTreeDepth, tree.Depth);
Assert.Equal(numberOfDocs, tree.NumberOfEntries);
Assert.Equal(expectedPageCount, tree.PageCount);
var hasSource = false;
List <ReduceTreePage> pages;
if (tree.Depth > 1)
{
// real tree
Assert.True(tree.Root.Children.Any());
Assert.Null(tree.Root.Entries);
pages = tree.Root.Children;
}
else
{
// nested section
Assert.Null(tree.Root.Children);
Assert.NotNull(tree.Root.Entries);
pages = new List <ReduceTreePage>
{
tree.Root
};
}
Assert.NotNull(tree.Root.AggregationResult);
foreach (var leafPage in pages)
{
Assert.Null(leafPage.Children);
Assert.NotNull(leafPage.AggregationResult);
foreach (var entry in leafPage.Entries)
{
if (string.IsNullOrEmpty(entry.Source) == false)
{
hasSource = true;
}
Assert.NotNull(entry.Data);
}
}
Assert.True(hasSource);
Assert.Equal(numberOfDocs, pages.Sum(x => x.Entries.Count));
}
}
}
}
}
}
}
public void CanSetMapBatchSize(int?mapBatchSize, int numberOfDocs)
{
var indexDefinition = new IndexDefinition
{
Name = "NewIndex",
Maps = new HashSet <string>
{
"from p in docs.Orders select new { CompanyName = LoadDocument(p.Company, \"Companies\").Name }"
},
Configuration = new IndexConfiguration
{
{
RavenConfiguration.GetKey(x => x.Indexing.MapBatchSize), mapBatchSize?.ToString()
}
}
};
using (var database = CreateDocumentDatabase())
using (var index = MapIndex.CreateNew(indexDefinition, database))
using (var queryContext = QueryOperationContext.ShortTermSingleUse(database))
{
var context = queryContext.Documents;
Assert.Equal(mapBatchSize, index.Configuration.MapBatchSize);
using (var tx = context.OpenWriteTransaction())
{
for (var i = 0; i < numberOfDocs; i++)
{
var orderDocumentId = $"orders/{i}";
var companyDocumentId = $"companies/{i}";
using (var doc = CreateDocument(context, orderDocumentId, new DynamicJsonValue
{
["Name"] = "John",
["Company"] = companyDocumentId,
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Orders"
}
}))
{
database.DocumentsStorage.Put(context, orderDocumentId, null, doc);
}
}
tx.Commit();
}
using (var tx = context.OpenWriteTransaction())
{
for (var i = 0; i < numberOfDocs; i++)
{
var companyDocumentId = $"companies/{i}";
using (var doc = CreateDocument(context, companyDocumentId, new DynamicJsonValue
{
["Name"] = "RavenDB",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Companies"
}
}))
{
database.DocumentsStorage.Put(context, companyDocumentId, null, doc);
}
}
tx.Commit();
}
var numberOfBatches = numberOfDocs / mapBatchSize;
var batchStats = new IndexingRunStats();
var stats = new IndexingStatsScope(batchStats);
for (var i = 0; i < numberOfBatches; i++)
{
index.DoIndexingWork(stats, CancellationToken.None);
Assert.Equal((i + 1) * mapBatchSize, stats.MapAttempts);
}
index.DoIndexingWork(stats, CancellationToken.None);
Assert.Equal(numberOfDocs, stats.MapAttempts);
using (var tx = context.OpenWriteTransaction())
{
for (var i = 0; i < numberOfDocs; i++)
{
var companyDocumentId = $"companies/{i}";
using (var doc = CreateDocument(context, companyDocumentId, new DynamicJsonValue
{
["Name"] = "Hibernating Rhinos",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Companies"
}
}))
{
database.DocumentsStorage.Put(context, companyDocumentId, null, doc);
}
}
tx.Commit();
}
batchStats = new IndexingRunStats();
stats = new IndexingStatsScope(batchStats);
for (var i = 0; i < numberOfBatches; i++)
{
index.DoIndexingWork(stats, CancellationToken.None);
Assert.Equal((i + 1) * mapBatchSize, stats.MapReferenceAttempts);
}
index.DoIndexingWork(stats, CancellationToken.None);
Assert.Equal(numberOfDocs, stats.MapReferenceAttempts);
using (context.OpenReadTransaction())
Assert.False(index.IsStale(queryContext));
}
}
public void StalenessCalculationShouldWorkForAllDocsIndexes()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapIndex.CreateNew(1, new IndexDefinition()
{
Name = "Index1",
Maps = { "from doc in docs select new { doc.Name }" },
Type = IndexType.Map
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
{
["Name"] = "John",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "users/1", null, doc);
}
using (var doc = CreateDocument(context, "people/1", new DynamicJsonValue
{
["Name"] = "Edward",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "People"
}
}))
{
database.DocumentsStorage.Put(context, "people/1", null, doc);
}
tx.Commit();
}
using (context.OpenReadTransaction())
{
var isStale = index.IsStale(context);
Assert.True(isStale);
}
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
index.DoIndexingWork(scope, CancellationToken.None);
Assert.Equal(2, batchStats.MapAttempts);
Assert.Equal(2, batchStats.MapSuccesses);
Assert.Equal(0, batchStats.MapErrors);
using (context.OpenReadTransaction())
{
var isStale = index.IsStale(context);
Assert.False(isStale);
}
using (var tx = context.OpenWriteTransaction())
{
database.DocumentsStorage.Delete(context, "people/1", null);
tx.Commit();
}
using (context.OpenReadTransaction())
{
var isStale = index.IsStale(context);
Assert.True(isStale);
}
batchStats = new IndexingRunStats();
scope = new IndexingStatsScope(batchStats);
index.DoIndexingWork(scope, CancellationToken.None);
using (context.OpenReadTransaction())
{
var isStale = index.IsStale(context);
Assert.False(isStale);
}
}
}
}
}
public async Task CleanupOfMultiMapIndexWithLoadDocument()
{
var indexDefinition = new IndexDefinition()
{
Name = "NewIndex",
Maps = new HashSet <string>
{
"from p in docs.Orders select new { CompanyName = LoadDocument(p.Company, \"Companies\").Name }",
"from p in docs.Companies select new { CompanyName = p.Name }"
}
};
using (var database = CreateDocumentDatabase())
using (var index = MapIndex.CreateNew(indexDefinition, database))
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "key/1", new DynamicJsonValue
{
["Name"] = "John",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Orders"
}
}))
{
database.DocumentsStorage.Put(context, "key/1", null, doc);
}
tx.Commit();
}
var batchStats = new IndexingRunStats();
var stats = new IndexingStatsScope(batchStats);
index.DoIndexingWork(stats, CancellationToken.None);
var tombstones = index.GetLastProcessedTombstonesPerCollection();
Assert.Equal(2, tombstones.Count);
Assert.Equal(0, tombstones["Orders"]);
Assert.Equal(0, tombstones["Companies"]);
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Orders", 0, 0, 128).Count();
Assert.Equal(0, count);
}
using (var tx = context.OpenWriteTransaction())
{
database.DocumentsStorage.Delete(context, "key/1", null);
tx.Commit();
}
tombstones = index.GetLastProcessedTombstonesPerCollection();
Assert.Equal(2, tombstones.Count);
Assert.Equal(0, tombstones["Orders"]);
Assert.Equal(0, tombstones["Companies"]);
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Orders", 0, 0, 128).Count();
Assert.Equal(1, count);
}
await database.TombstoneCleaner.ExecuteCleanup();
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Orders", 0, 0, 128).Count();
Assert.Equal(1, count);
}
batchStats = new IndexingRunStats();
stats = new IndexingStatsScope(batchStats);
index.DoIndexingWork(stats, CancellationToken.None);
tombstones = index.GetLastProcessedTombstonesPerCollection();
Assert.Equal(2, tombstones.Count);
Assert.Equal(2, tombstones["Orders"]);
Assert.Equal(0, tombstones["Companies"]);
await database.TombstoneCleaner.ExecuteCleanup();
using (context.OpenReadTransaction())
{
var list = database.DocumentsStorage.GetTombstonesFrom(context, "Orders", 0, 0, 128).ToList();
Assert.Equal(0, list.Count);
}
}
}
public async Task GetErrors()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Fitzchak" });
await session.StoreAsync(new User { Name = "Arek" });
await session.SaveChangesAsync();
}
using (var session = store.OpenSession())
{
var users = session
.Query <User>()
.Customize(x => x.WaitForNonStaleResults())
.Where(x => x.Name == "Arek")
.ToList();
Assert.Equal(1, users.Count);
}
var database = await Server.ServerStore.DatabasesLandlord
.TryGetOrCreateResourceStore(new StringSegment(store.Database))
;
var index = database.IndexStore.GetIndexes().First();
var now = SystemTime.UtcNow;
var nowNext = now.AddTicks(1);
var batchStats = new IndexingRunStats();
batchStats.AddMapError("users/1", "error/1");
batchStats.AddAnalyzerError(new IndexAnalyzerException());
batchStats.Errors[0].Timestamp = now;
batchStats.Errors[1].Timestamp = nowNext;
index._indexStorage.UpdateStats(SystemTime.UtcNow, batchStats);
var errors = await store.Maintenance.SendAsync(new GetIndexErrorsOperation(new[] { index.Name }));
var error = errors[0];
Assert.Equal(index.Name, error.Name);
Assert.Equal(2, error.Errors.Length);
Assert.Equal("Map", error.Errors[0].Action);
Assert.Equal("users/1", error.Errors[0].Document);
Assert.Equal("error/1", error.Errors[0].Error);
Assert.Equal(now, error.Errors[0].Timestamp);
Assert.Equal("Analyzer", error.Errors[1].Action);
Assert.Null(error.Errors[1].Document);
Assert.True(error.Errors[1].Error.Contains("Could not create analyzer:"));
Assert.Equal(nowNext, error.Errors[1].Timestamp);
errors = await store.Maintenance.SendAsync(new GetIndexErrorsOperation());
Assert.Equal(1, errors.Length);
errors = await store.Maintenance.SendAsync(new GetIndexErrorsOperation(new[] { index.Name }));
Assert.Equal(1, errors.Length);
var stats = await store.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.Name));
Assert.Equal(2, stats.ErrorsCount);
}
}
public async Task By_complex_object_and_array()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition()
{
Name = "Users_GroupByLocationAndResidenceAddress",
Maps = { @"from user in docs.Users select new {
user.Hobbies,
user.ResidenceAddress,
Count = 1
}" },
Reduce = @"from result in results group result by new { result.Hobbies, result.ResidenceAddress } into g select new {
g.Key.Hobbies,
g.Key.ResidenceAddress,
Count = g.Sum(x => x.Count)
}",
Fields = new Dictionary <string, IndexFieldOptions>()
{
{ "Hobbies", new IndexFieldOptions()
{
Indexing = FieldIndexing.Search,
} },
{ "ResidenceAddress", new IndexFieldOptions()
{
Indexing = FieldIndexing.Search,
} }
}
}, database))
{
DocumentQueryResult queryResult;
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
Put_docs(context, database);
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
while (index.DoIndexingWork(scope, CancellationToken.None))
{
}
queryResult =
await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Hobbies IN ('music')"), context, OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(1, results.Count);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal("music", ((BlittableJsonReaderArray)results[0].Data["Hobbies"])[0].ToString());
Assert.Equal("sport", ((BlittableJsonReaderArray)results[0].Data["Hobbies"])[1].ToString());
Assert.Equal(@"{""Country"":""UK""}", results[0].Data["ResidenceAddress"].ToString());
Assert.Equal(2L, results[0].Data["Count"]);
}
}
}
}
public void When_map_results_do_not_change_then_we_skip_the_reduce_phase(int numberOfDocs)
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
{
Name = "Users_ByCount_GroupByProduct",
Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
Product = g.Key,
Count = g.Sum(x=> x.Count),
Total = g.Sum(x=> x.Total)
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
PutOrder(database, order, context, i);
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
while (index.DoIndexingWork(scope, cts.Token))
{
;
}
Assert.Equal(numberOfDocs, firstRunStats.MapAttempts);
Assert.Equal(numberOfDocs, firstRunStats.MapSuccesses);
Assert.Equal(0, firstRunStats.MapErrors);
Assert.True(firstRunStats.ReduceAttempts > 0);
Assert.True(firstRunStats.ReduceSuccesses > 0);
Assert.Equal(0, firstRunStats.ReduceErrors);
for (int i = 0; i < numberOfDocs; i++)
{
var order = CreateOrder();
order["RefNumber"] = "456";
PutOrder(database, order, context, i);
}
var secondRunStats = new IndexingRunStats();
scope = new IndexingStatsScope(secondRunStats);
cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
while (index.DoIndexingWork(scope, cts.Token))
{
;
}
Assert.Equal(firstRunStats.MapAttempts, secondRunStats.MapAttempts);
Assert.Equal(firstRunStats.MapSuccesses, secondRunStats.MapSuccesses);
Assert.Equal(0, secondRunStats.MapErrors);
Assert.Equal(0, secondRunStats.ReduceAttempts);
Assert.Equal(0, secondRunStats.ReduceSuccesses);
Assert.Equal(0, secondRunStats.ReduceErrors);
}
}
}
}
public async Task By_multiple_complex_objects()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
{
Name = "Users_GroupByLocationAndResidenceAddress",
Maps = { @"from user in docs.Users select new {
user.Location,
user.ResidenceAddress,
Count = 1
}" },
Reduce = @"from result in results group result by new { result.Location, result.ResidenceAddress } into g select new {
g.Key.Location,
g.Key.ResidenceAddress,
Count = g.Sum(x => x.Count)
}",
Fields = new Dictionary <string, IndexFieldOptions>()
{
{ "Location", new IndexFieldOptions()
{
Indexing = FieldIndexing.Search,
} },
{ "ResidenceAddress", new IndexFieldOptions()
{
Indexing = FieldIndexing.Search,
} }
}
}, database))
{
DocumentQueryResult queryResult;
using (var context = QueryOperationContext.ShortTermSingleUse(database))
{
Put_docs(context.Documents, database);
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
while (index.DoIndexingWork(scope, CancellationToken.None))
{
}
queryResult =
await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
}
using (var context = QueryOperationContext.ShortTermSingleUse(database))
{
queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Location = 'Poland'"), context, OperationCancelToken.None);
var results = queryResult.Results;
Assert.Equal(1, results.Count);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal(@"{""Country"":""Poland"",""State"":""Pomerania""}", results[0].Data["Location"].ToString());
Assert.Equal(@"{""Country"":""UK""}", results[0].Data["ResidenceAddress"].ToString());
Assert.Equal(2L, results[0].Data["Count"]);
}
}
}
}
public async Task The_easiest_static_index()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapIndex.CreateNew(1, new IndexDefinition()
{
Name = "Users_ByName",
Maps = { "from user in docs.Users select new { user.Name }" },
Type = IndexType.Map
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
{
["Name"] = "John",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "users/1", null, doc);
}
using (var doc = CreateDocument(context, "users/2", new DynamicJsonValue
{
["Name"] = "Edward",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "users/2", null, doc);
}
tx.Commit();
}
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
index.DoIndexingWork(scope, CancellationToken.None);
Assert.Equal(2, batchStats.MapAttempts);
Assert.Equal(2, batchStats.MapSuccesses);
Assert.Equal(0, batchStats.MapErrors);
var queryResult = await index.Query(new IndexQueryServerSide(), context, OperationCancelToken.None);
Assert.Equal(2, queryResult.Results.Count);
context.ResetAndRenew();
queryResult = await index.Query(new IndexQueryServerSide()
{
Query = "Name:John"
}, context, OperationCancelToken.None);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal("users/1", queryResult.Results[0].Key);
}
}
}
}
public async Task Cleanup()
{
using (var database = CreateDocumentDatabase())
using (var index = AutoMapIndex.CreateNew(new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
}
}), database))
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "key/1", new DynamicJsonValue
{
["Name"] = "John",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "key/1", null, doc);
}
using (var doc = CreateDocument(context, "key/2", new DynamicJsonValue
{
["Name"] = "Edward",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "key/2", null, doc);
}
using (var doc = CreateDocument(context, "key/3", new DynamicJsonValue
{
["Name"] = "William",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "key/3", null, doc);
}
tx.Commit();
}
var batchStats = new IndexingRunStats();
var stats = new IndexingStatsScope(batchStats);
index.DoIndexingWork(stats, CancellationToken.None);
var tombstones = index.GetLastProcessedTombstonesPerCollection();
Assert.Equal(1, tombstones.Count);
Assert.Equal(0, tombstones["Users"]);
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Users", 0, 0, 128).Count();
Assert.Equal(0, count);
}
await database.TombstoneCleaner.ExecuteCleanup();
using (var tx = context.OpenWriteTransaction())
{
database.DocumentsStorage.Delete(context, "key/1", null);
tx.Commit();
}
tombstones = index.GetLastProcessedTombstonesPerCollection();
Assert.Equal(1, tombstones.Count);
Assert.Equal(0, tombstones["Users"]);
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Users", 0, 0, 128).Count();
Assert.Equal(1, count);
}
await database.TombstoneCleaner.ExecuteCleanup();
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Users", 0, 0, 128).Count();
Assert.Equal(1, count);
}
batchStats = new IndexingRunStats();
stats = new IndexingStatsScope(batchStats);
index.DoIndexingWork(stats, CancellationToken.None);
tombstones = index.GetLastProcessedTombstonesPerCollection();
Assert.Equal(1, tombstones.Count);
Assert.Equal(4, tombstones["Users"]);
using (var tx = context.OpenWriteTransaction())
{
database.DocumentsStorage.Delete(context, "key/2", null);
tx.Commit();
}
using (context.OpenReadTransaction())
{
var count = database.DocumentsStorage.GetTombstonesFrom(context, "Users", 0, 0, 128).Count();
Assert.Equal(2, count);
}
await database.TombstoneCleaner.ExecuteCleanup();
using (context.OpenReadTransaction())
{
var list = database.DocumentsStorage.GetTombstonesFrom(context, "Users", 0, 0, 128).ToList();
Assert.Equal(1, list.Count);
Assert.Equal(5, list[0].Etag);
}
}
}
public void NumberOfDocumentsAndTombstonesToProcessShouldBeCalculatedCorrectly()
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapIndex.CreateNew(1, new IndexDefinition()
{
Name = "Index1",
Maps = { "from doc in docs.Users select new { doc.Name }" },
Type = IndexType.Map
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
using (var tx = context.OpenWriteTransaction())
{
using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
{
["Name"] = "John",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "users/1", null, doc);
}
using (var doc = CreateDocument(context, "users/2", new DynamicJsonValue
{
["Name"] = "Bob",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "users/2", null, doc);
}
using (var doc = CreateDocument(context, "people/1", new DynamicJsonValue
{
["Name"] = "Edward",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "People"
}
}))
{
database.DocumentsStorage.Put(context, "people/1", null, doc);
}
tx.Commit();
}
IndexProgress progress;
using (context.OpenReadTransaction())
{
progress = index.GetProgress(context);
}
Assert.Equal(0, progress.Collections["Users"].LastProcessedDocumentEtag);
Assert.Equal(0, progress.Collections["Users"].LastProcessedTombstoneEtag);
Assert.Equal(2, progress.Collections["Users"].NumberOfDocumentsToProcess);
Assert.Equal(0, progress.Collections["Users"].NumberOfTombstonesToProcess);
Assert.Equal(2, progress.Collections["Users"].TotalNumberOfDocuments);
Assert.Equal(0, progress.Collections["Users"].TotalNumberOfTombstones);
var batchStats = new IndexingRunStats();
var scope = new IndexingStatsScope(batchStats);
index.DoIndexingWork(scope, CancellationToken.None);
using (context.OpenReadTransaction())
{
progress = index.GetProgress(context);
}
Assert.Equal(2, progress.Collections["Users"].LastProcessedDocumentEtag);
Assert.Equal(0, progress.Collections["Users"].LastProcessedTombstoneEtag);
Assert.Equal(0, progress.Collections["Users"].NumberOfDocumentsToProcess);
Assert.Equal(0, progress.Collections["Users"].NumberOfTombstonesToProcess);
Assert.Equal(2, progress.Collections["Users"].TotalNumberOfDocuments);
Assert.Equal(0, progress.Collections["Users"].TotalNumberOfTombstones);
using (var tx = context.OpenWriteTransaction())
{
database.DocumentsStorage.Delete(context, "users/1", null);
using (var doc = CreateDocument(context, "users/3", new DynamicJsonValue
{
["Name"] = "George",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "Users"
}
}))
{
database.DocumentsStorage.Put(context, "users/3", null, doc);
}
using (var doc = CreateDocument(context, "people/2", new DynamicJsonValue
{
["Name"] = "Edward",
[Constants.Metadata.Key] = new DynamicJsonValue
{
[Constants.Headers.RavenEntityName] = "People"
}
}))
{
database.DocumentsStorage.Put(context, "people/2", null, doc);
}
tx.Commit();
}
using (context.OpenReadTransaction())
{
progress = index.GetProgress(context);
}
Assert.Equal(2, progress.Collections["Users"].LastProcessedDocumentEtag);
Assert.Equal(0, progress.Collections["Users"].LastProcessedTombstoneEtag);
Assert.Equal(1, progress.Collections["Users"].NumberOfDocumentsToProcess);
Assert.Equal(1, progress.Collections["Users"].NumberOfTombstonesToProcess);
Assert.Equal(2, progress.Collections["Users"].TotalNumberOfDocuments);
Assert.Equal(1, progress.Collections["Users"].TotalNumberOfTombstones);
batchStats = new IndexingRunStats();
scope = new IndexingStatsScope(batchStats);
index.DoIndexingWork(scope, CancellationToken.None);
using (context.OpenReadTransaction())
{
progress = index.GetProgress(context);
}
Assert.Equal(5, progress.Collections["Users"].LastProcessedDocumentEtag);
Assert.Equal(4, progress.Collections["Users"].LastProcessedTombstoneEtag);
Assert.Equal(0, progress.Collections["Users"].NumberOfDocumentsToProcess);
Assert.Equal(0, progress.Collections["Users"].NumberOfTombstonesToProcess);
Assert.Equal(2, progress.Collections["Users"].TotalNumberOfDocuments);
Assert.Equal(1, progress.Collections["Users"].TotalNumberOfTombstones);
}
}
}
}
public async Task Reduction_should_ignore_overflow_pages(long numberOfDocs)
{
using (var database = CreateDocumentDatabase())
{
using (var index = MapReduceIndex.CreateNew(new IndexDefinition()
{
Name = "Users_ByLocation",
Maps = { @"from user in docs.Users
select new { Location = user.Location, Count = 1 }" },
Reduce = @"from result in results
group result by result.Location into g
select new
{
Location = g.Key,
Count = g.Sum(x=> x.Count)
}",
}, database))
{
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
{
var bytes = new byte[4096];
new Random(2).NextBytes(bytes); // TODO arek - seed
var randomLocation = Encoding.ASCII.GetString(bytes);
using (var tx = context.OpenWriteTransaction())
{
for (int i = 0; i < numberOfDocs; i++)
{
var user = new DynamicJsonValue()
{
["Location"] = randomLocation,
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
};
using (var doc = CreateDocument(context, $"users/{i}", user))
{
database.DocumentsStorage.Put(context, $"users/{i}", null, doc);
}
}
tx.Commit();
}
var firstRunStats = new IndexingRunStats();
var scope = new IndexingStatsScope(firstRunStats);
index.DoIndexingWork(scope, CancellationToken.None);
Assert.Equal(numberOfDocs, firstRunStats.MapAttempts);
Assert.Equal(numberOfDocs, firstRunStats.MapSuccesses);
Assert.Equal(0, firstRunStats.MapErrors);
var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);
Assert.False(queryResult.IsStale);
Assert.Equal(1, queryResult.Results.Count);
Assert.Equal(numberOfDocs, queryResult.Results[0].Data["Count"]);
}
}
}
}
