Exemplo n.º 1
0
        public async Task Should_use_auto_index_even_if_idle_when_match_is_complete()
        {
            using (var database = CreateDocumentDatabase())
            {
                var autoIndex = await database.IndexStore.CreateIndex(new AutoMapIndexDefinition("Users", new[]
                {
                    new AutoIndexField
                    {
                        Name = "FirstName",
                    },
                    new AutoIndexField
                    {
                        Name = "LastName",
                    }
                }), Guid.NewGuid().ToString());

                autoIndex.SetState(IndexState.Idle);

                using (var context = QueryOperationContext.ShortTermSingleUse(database))
                {
                    // it shouldn't throw
                    await database.QueryRunner.ExecuteQuery(new IndexQueryServerSide("from Users where LastName = 'Arek'"), context, null,
                                                            OperationCancelToken.None);
                }

                var sameIndex = database.IndexStore.GetIndex(autoIndex.Name);

                Assert.Same(autoIndex, sameIndex);
                Assert.Equal(IndexState.Normal, sameIndex.State);
            }
        }
Exemplo n.º 2
0
        public async Task By_single_array_object()
        {
            using (var database = CreateDocumentDatabase())
            {
                using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
                {
                    Name = "Users_GroupByHobbies",
                    Maps = { @"from user in docs.Users select new { 
                                user.Hobbies,
                                Count = 1
                            }" },
                    Reduce = @"from result in results group result by result.Hobbies into g select new { 
                                Hobbies = g.Key, 
                                Count = g.Sum(x => x.Count)
                            }",
                    Fields = new Dictionary <string, IndexFieldOptions>()
                    {
                        { "Hobbies", new IndexFieldOptions()
                          {
                              Indexing = FieldIndexing.Search,
                          } }
                    }
                }, database))
                {
                    DocumentQueryResult queryResult;
                    using (var context = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        Put_docs(context.Documents, database);

                        var batchStats = new IndexingRunStats();
                        var scope      = new IndexingStatsScope(batchStats);
                        index.DoIndexingWork(scope, CancellationToken.None);

                        queryResult =
                            await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);

                        Assert.Equal(2, queryResult.Results.Count);
                    }
                    using (var context = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Hobbies = 'music'"), context, OperationCancelToken.None);

                        var results = queryResult.Results;

                        Assert.Equal(1, results.Count);

                        Assert.Equal(1, queryResult.Results.Count);
                        Assert.Equal("music", ((BlittableJsonReaderArray)results[0].Data["Hobbies"])[0].ToString());
                        Assert.Equal("sport", ((BlittableJsonReaderArray)results[0].Data["Hobbies"])[1].ToString());
                        Assert.Equal(2L, results[0].Data["Count"]);

                        foreach (var document in results)
                        {
                            document.Data.Dispose();
                        }
                    }
                }
            }
        }
Exemplo n.º 3
0
        public async Task CanGroupByNestedFieldAndAggregateOnCollection()
        {
            using (var db = CreateDocumentDatabase())
                using (var mri = AutoMapReduceIndex.CreateNew(new AutoMapReduceIndexDefinition(
                                                                  "Orders",
                                                                  new[]
                {
                    new AutoIndexField
                    {
                        Name = "Lines[].Quantity",
                        Aggregation = AggregationOperation.Sum,
                        Storage = FieldStorage.Yes
                    },
                    new AutoIndexField
                    {
                        Name = "Lines[].Price",
                        Aggregation = AggregationOperation.Sum,
                        Storage = FieldStorage.Yes
                    }
                },
                                                                  new[]
                {
                    new AutoIndexField
                    {
                        Name = "ShipTo.Country",
                        Storage = FieldStorage.Yes
                    },
                }), db))
                {
                    CreateOrders(db, 5, new[] { "Poland", "Israel" });

                    mri.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await mri.Query(new IndexQueryServerSide($"FROM INDEX '{mri.Name}' WHERE ShipTo.Country = 'Poland'"), context, OperationCancelToken.None);

                        Assert.Equal(1, queryResult.Results.Count);
                        var result = queryResult.Results[0].Data;

                        string location;
                        Assert.True(result.TryGet("ShipTo.Country", out location));
                        Assert.Equal("Poland", location);

                        var price = result["Lines[].Price"] as LazyNumberValue;

                        Assert.NotNull(price);

                        Assert.Equal(63.6, price, 1);

                        var quantity = result["Lines[].Quantity"];

                        Assert.Equal(9L, quantity);
                    }
                }
        }
Exemplo n.º 4
0
        public async Task By_single_complex_object()
        {
            using (var database = CreateDocumentDatabase())
            {
                using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
                {
                    Name = "Users_ByCount_GroupByLocation",
                    Maps = { @"from user in docs.Users select new { 
                                user.Location,
                                Count = 1
                            }" },
                    Reduce = @"from result in results group result by result.Location into g select new { 
                                Location = g.Key, 
                                Count = g.Sum(x => x.Count)
                            }",
                    Fields = new Dictionary <string, IndexFieldOptions>()
                    {
                        { "Location", new IndexFieldOptions()
                          {
                              Indexing = FieldIndexing.Search,
                          } }
                    }
                }, database))
                {
                    DocumentQueryResult queryResult;
                    using (var context = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        Put_docs(context.Documents, database);

                        var batchStats = new IndexingRunStats();
                        var scope      = new IndexingStatsScope(batchStats);
                        index.DoIndexingWork(scope, CancellationToken.None);

                        queryResult =
                            await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);

                        Assert.Equal(2, queryResult.Results.Count);
                    }
                    using (var context = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Location = 'Poland'"), context, OperationCancelToken.None);

                        var results = queryResult.Results;

                        Assert.Equal(1, results.Count);

                        Assert.Equal(1, queryResult.Results.Count);
                        Assert.Equal(@"{""Country"":""Poland"",""State"":""Pomerania""}", results[0].Data["Location"].ToString());
                        Assert.Equal(2L, results[0].Data["Count"]);
                    }
                }
            }
        }
Exemplo n.º 5
0
        public async Task PatchByIndex_WhenFinish_ShouldFreeInternalUsageMemory()
        {
            using (var store = GetDocumentStore())
            {
                var database = await GetDatabase(store.Database);

                var index = new IndexDefinition
                {
                    Name = "Users_ByName",
                    Maps = { "from user in docs.Users select new { user.Name }" },
                    Type = IndexType.Map
                };

                await store
                .Maintenance
                .SendAsync(new PutIndexesOperation(new[] { index }));

                using (var session = store.OpenAsyncSession())
                {
                    for (var i = 0; i < 100; i++)
                    {
                        await session.StoreAsync(new User { Name = "John" });
                    }
                    await session.SaveChangesAsync();
                }
                WaitForIndexing(store);

                using (var context = QueryOperationContext.ShortTermSingleUse(database))
                {
                    var query = new IndexQueryServerSide($"FROM index '{index.Name}'");
                    var patch = new PatchRequest("var u = this; u.is = true;", PatchRequestType.Patch, query.Metadata.DeclaredFunctions);

                    var before = context.Documents.AllocatedMemory;
                    await database.QueryRunner.ExecutePatchQuery(
                        query,
                        new QueryOperationOptions { RetrieveDetails = true },
                        patch,
                        query.QueryParameters,
                        context,
                        p => { },
                        new OperationCancelToken(CancelAfter, CancellationToken.None, CancellationToken.None));

                    var after = context.Documents.AllocatedMemory;

                    //In a case of fragmentation, we don't immediately freeing memory so the memory can be a little bit higher
                    const long threshold = 256;
                    Assert.True(Math.Abs(before - after) < threshold);
                }
            }
        }
Exemplo n.º 6
0
        public async Task MultipleReduceKeys(int numberOfUsers, string[] locations)
        {
            using (var db = CreateDocumentDatabase())
                using (var index = AutoMapReduceIndex.CreateNew(GetUsersCountByLocationIndexDefinition(), db))
                {
                    CreateUsers(db, numberOfUsers, locations);

                    var batchStats = new IndexingRunStats();
                    var scope      = new IndexingStatsScope(batchStats);

                    var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
                    while (index.DoIndexingWork(scope, cts.Token))
                    {
                        ;
                    }

                    Assert.Equal(numberOfUsers, batchStats.MapAttempts);
                    Assert.Equal(numberOfUsers, batchStats.MapSuccesses);
                    Assert.Equal(0, batchStats.MapErrors);
                    Assert.True(batchStats.ReduceAttempts >= numberOfUsers, $"{batchStats.ReduceAttempts} >= {numberOfUsers}");
                    Assert.True(batchStats.ReduceSuccesses >= numberOfUsers, $"{batchStats.ReduceSuccesses} >= {numberOfUsers}");
                    Assert.Equal(batchStats.ReduceAttempts, batchStats.ReduceSuccesses);
                    Assert.Equal(0, batchStats.ReduceErrors);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'")
                        {
                            WaitForNonStaleResultsTimeout = TimeSpan.FromMinutes(1)
                        }, context, OperationCancelToken.None);

                        Assert.False(queryResult.IsStale);

                        var results = queryResult.Results;

                        Assert.Equal(locations.Length, results.Count);

                        for (int i = 0; i < locations.Length; i++)
                        {
                            Assert.Equal(locations[i], results[i].Data["Location"].ToString());

                            long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
                            Assert.Equal(expected, results[i].Data["Count"]);
                        }
                    }
                }
        }
Exemplo n.º 7
0
    public async Task Should_throw_on_attempt_to_create_auto_index()
    {
        using (var database = CreateDocumentDatabase())
        {
            using (var context = QueryOperationContext.ShortTermSingleUse(database))
            {
                var ex = await Assert.ThrowsAsync <InvalidOperationException>(async() =>
                {
                    await database.QueryRunner.ExecuteQuery(new IndexQueryServerSide("from Users where LastName = 'Arek'")
                    {
                        DisableAutoIndexCreation = true
                    }, context, null,
                                                            OperationCancelToken.None);
                });

                Assert.Equal("Creation of Auto Indexes was disabled and no Auto Index matching the given query was found.", ex.Message);
            }
        }
    }
Exemplo n.º 8
0
        public async Task CanUseSimpleReduction()
        {
            using (var db = CreateDocumentDatabase())
                using (var mri = AutoMapReduceIndex.CreateNew(GetUsersCountByLocationIndexDefinition(), db))
                {
                    CreateUsers(db, 2, "Poland");

                    mri.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await mri.Query(new IndexQueryServerSide($"FROM INDEX '{mri.Name}'"), context, OperationCancelToken.None);

                        Assert.Equal(1, queryResult.Results.Count);
                        var result = queryResult.Results[0].Data;

                        string location;
                        Assert.True(result.TryGet("Location", out location));
                        Assert.Equal("Poland", location);

                        var count = result["Count"];

                        Assert.Equal(2L, count);
                    }

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await mri.Query(new IndexQueryServerSide($"FROM INDEX '{mri.Name}' WHERE Count BETWEEN 2 AND 10"), context,
                                                          OperationCancelToken.None);

                        Assert.Equal(1, queryResult.Results.Count);
                    }
                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await mri.Query(new IndexQueryServerSide($"FROM INDEX '{mri.Name}' WHERE Count >= 10"), context, OperationCancelToken.None);

                        Assert.Equal(0, queryResult.Results.Count);
                    }
                }
        }
Exemplo n.º 9
0
        public async Task CanUpdateByChangingReduceKey()
        {
            using (var db = CreateDocumentDatabase())
                using (var index = AutoMapReduceIndex.CreateNew(new AutoMapReduceIndexDefinition("Users", new[]
                {
                    new AutoIndexField
                    {
                        Name = "Age",
                        Aggregation = AggregationOperation.Sum,
                        Storage = FieldStorage.Yes
                    }
                }, new[]
                {
                    new AutoIndexField
                    {
                        Name = "Location",
                        Storage = FieldStorage.Yes
                    },
                }), db))
                {
                    CreateUsers(db, 2, "Poland");

                    index.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);

                        var results = queryResult.Results;

                        Assert.Equal(1, results.Count);

                        Assert.Equal("Poland", results[0].Data["Location"].ToString());
                        Assert.Equal(41L, results[0].Data["Age"]);
                    }

                    using (var context = DocumentsOperationContext.ShortTermSingleUse(db))
                    {
                        using (var tx = context.OpenWriteTransaction())
                        {
                            using (var doc = context.ReadObject(new DynamicJsonValue
                            {
                                ["Name"] = "James",
                                ["Location"] = "Israel",
                                ["Age"] = 20,
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Users"
                                }
                            }, "users/0"))
                            {
                                db.DocumentsStorage.Put(context, "users/0", null, doc);
                            }

                            tx.Commit();
                        }
                    }

                    index.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await index.Query(new IndexQueryServerSide("FROM Users ORDER BY Location"), context, OperationCancelToken.None);

                        var results = queryResult.Results;

                        Assert.Equal(2, results.Count);

                        Assert.Equal("Israel", results[0].Data["Location"].ToString());
                        Assert.Equal(20L, results[0].Data["Age"]);

                        Assert.Equal("Poland", results[1].Data["Location"].ToString());
                        Assert.Equal(21L, results[1].Data["Age"]);
                    }
                }
        }
Exemplo n.º 10
0
        public async Task The_simpliest_static_map_reduce_index()
        {
            using (var database = CreateDocumentDatabase())
            {
                using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
                {
                    Name = "Users_ByCount_GroupByLocation",
                    Maps = { @"from user in docs.Users select new { 
                                user.Location, 
                                CountInteger = 1, 
                                CountDouble = 1.0,
                                CastedInteger = 1
                            }" },
                    Reduce = @"from result in results group result by result.Location into g select new { 
                                Location = g.Key, 
                                CountInteger = g.Sum(x => x.CountInteger), 
                                CountDouble = g.Sum(x => x.CountDouble),
                                CastedInteger = g.Sum(x => (int)x.CastedInteger) 
                            }"
                }, database))
                {
                    DocumentQueryResult queryResult;
                    using (var queryContext = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        var context = queryContext.Documents;

                        using (var tx = context.OpenWriteTransaction())
                        {
                            using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
                            {
                                ["Location"] = "Poland",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Users"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "users/1", null, doc);
                            }

                            using (var doc = CreateDocument(context, "users/2", new DynamicJsonValue
                            {
                                ["Location"] = "Poland",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Users"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "users/2", null, doc);
                            }

                            tx.Commit();
                        }

                        var batchStats = new IndexingRunStats();
                        var scope      = new IndexingStatsScope(batchStats);
                        index.DoIndexingWork(scope, CancellationToken.None);

                        Assert.Equal(2, batchStats.MapAttempts);
                        Assert.Equal(2, batchStats.MapSuccesses);
                        Assert.Equal(0, batchStats.MapErrors);

                        Assert.Equal(2, batchStats.ReduceAttempts);
                        Assert.Equal(2, batchStats.ReduceSuccesses);
                        Assert.Equal(0, batchStats.ReduceErrors);

                        queryResult =
                            await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), queryContext, OperationCancelToken.None);

                        Assert.Equal(1, queryResult.Results.Count);
                    }
                    using (var context = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Location = 'Poland'"), context, OperationCancelToken.None);

                        var results = queryResult.Results;

                        Assert.Equal(1, results.Count);

                        Assert.Equal(1, queryResult.Results.Count);
                        Assert.Equal("Poland", results[0].Data["Location"].ToString());
                        Assert.Equal(2L, results[0].Data["CountInteger"]);
                        Assert.Equal(2.0, (LazyNumberValue)results[0].Data["CountDouble"]);
                        Assert.Equal(2L, results[0].Data["CastedInteger"]);
                    }
                }
            }
        }
Exemplo n.º 11
0
        private static async Task ActualTest(int numberOfUsers, string[] locations, Index index,
                                             MapReduceIndexingContext mapReduceContext, IIndexingWork reducer, DocumentDatabase database)
        {
            TransactionOperationContext indexContext;

            using (index._contextPool.AllocateOperationContext(out indexContext))
            {
                ulong hashOfReduceKey = 73493;

                using (var tx = indexContext.OpenWriteTransaction())
                {
                    mapReduceContext.MapPhaseTree    = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
                    mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);

                    var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);

                    for (int i = 0; i < numberOfUsers; i++)
                    {
                        using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
                        {
                            ["Count"] = 1,
                            ["Location"] = locations[i % locations.Length]
                        }, $"users/{i}"))
                        {
                            store.Add(i, mappedResult);
                        }
                    }

                    mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);

                    var writeOperation =
                        new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, null));

                    var stats = new IndexingStatsScope(new IndexingRunStats());
                    reducer.Execute(null, indexContext,
                                    writeOperation,
                                    stats, CancellationToken.None);

                    using (var indexWriteOperation = writeOperation.Value)
                    {
                        indexWriteOperation.Commit(stats);
                    }

                    index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);

                    mapReduceContext.Dispose();

                    tx.Commit();
                }

                using (var termSingleUse = QueryOperationContext.ShortTermSingleUse(database))
                {
                    var queryResult = await
                                      index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"),
                                                  termSingleUse,
                                                  OperationCancelToken.None);

                    var results = queryResult.Results;

                    Assert.Equal(locations.Length, results.Count);

                    for (int i = 0; i < locations.Length; i++)
                    {
                        Assert.Equal(locations[i], results[i].Data["Location"].ToString());

                        long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
                        Assert.Equal(expected, results[i].Data["Count"]);
                    }
                }

                // update

                using (var tx = indexContext.OpenWriteTransaction())
                {
                    mapReduceContext.MapPhaseTree    = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
                    mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);

                    var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);

                    for (int i = 0; i < locations.Length; i++)
                    {
                        using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
                        {
                            ["Count"] = 2, // increased by 1
                            ["Location"] = locations[i % locations.Length]
                        }, $"users/{i}"))
                        {
                            store.Add(i, mappedResult);
                        }
                    }

                    mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);

                    var writeOperation =
                        new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, null));
                    try
                    {
                        var stats = new IndexingStatsScope(new IndexingRunStats());
                        reducer.Execute(null, indexContext,
                                        writeOperation,
                                        stats, CancellationToken.None);

                        using (var indexWriteOperation = writeOperation.Value)
                        {
                            indexWriteOperation.Commit(stats);
                        }

                        index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);

                        mapReduceContext.Dispose();
                    }
                    finally
                    {
                        if (writeOperation.IsValueCreated)
                        {
                            writeOperation.Value.Dispose();
                        }
                    }

                    tx.Commit();
                }

                using (var shortTermSingleUse = QueryOperationContext.ShortTermSingleUse(database))
                {
                    var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"),
                                                        shortTermSingleUse,
                                                        OperationCancelToken.None);


                    var results = queryResult.Results;

                    Assert.Equal(locations.Length, results.Count);

                    for (int i = 0; i < locations.Length; i++)
                    {
                        Assert.Equal(locations[i], results[i].Data["Location"].ToString());

                        long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
                        Assert.Equal(expected + 1, results[i].Data["Count"]);
                    }
                }
                // delete

                using (var tx = indexContext.OpenWriteTransaction())
                {
                    mapReduceContext.MapPhaseTree    = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
                    mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);

                    var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);

                    for (int i = 0; i < locations.Length; i++)
                    {
                        store.Delete(i);
                    }

                    mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);

                    var writeOperation =
                        new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, null));
                    try
                    {
                        var stats = new IndexingStatsScope(new IndexingRunStats());
                        reducer.Execute(null, indexContext,
                                        writeOperation,
                                        stats, CancellationToken.None);

                        using (var indexWriteOperation = writeOperation.Value)
                        {
                            indexWriteOperation.Commit(stats);
                        }

                        index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);

                        tx.Commit();
                    }
                    finally
                    {
                        if (writeOperation.IsValueCreated)
                        {
                            writeOperation.Value.Dispose();
                        }
                    }
                }

                using (var documentsOperationContext = QueryOperationContext.ShortTermSingleUse(database))
                {
                    var queryResult = await index.Query(new IndexQueryServerSide("FROM Users ORDER BY Location"),
                                                        documentsOperationContext,
                                                        OperationCancelToken.None);


                    var results = queryResult.Results;

                    Assert.Equal(locations.Length, results.Count);

                    for (int i = 0; i < locations.Length; i++)
                    {
                        Assert.Equal(locations[i], results[i].Data["Location"].ToString());

                        long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
                        Assert.Equal(expected - 1, results[i].Data["Count"]);
                    }
                }
            }
        }
Exemplo n.º 12
0
        public void Should_be_able_to_read_index_stats_even_if_corruption_happened()
        {
            UseNewLocalServer();

            using (var db = CreateDocumentDatabase())
            {
                using (var index = MapIndex.CreateNew(new IndexDefinition()
                {
                    Name = "Users_ByName",
                    Maps =
                    {
                        "from user in docs.Users select new { user.Name }"
                    },
                    Type = IndexType.Map
                }, db))
                {
                    PutUser(db);

                    index._indexStorage.SimulateCorruption = true;

                    db.ServerStore.DatabasesLandlord.CatastrophicFailureHandler.MaxDatabaseUnloads = 0;

                    var mre = new ManualResetEventSlim();

                    db.Changes.OnIndexChange += change =>
                    {
                        if (change.Type == IndexChangeTypes.IndexMarkedAsErrored)
                        {
                            mre.Set();
                        }
                    };

                    index.Start();

                    Assert.True(mre.Wait(TimeSpan.FromMinutes(1)));
                    Assert.Equal(IndexState.Error, index.State);

                    long errorCount = 0;

                    for (int i = 0; i < 20; i++)
                    {
                        errorCount = index.GetErrorCount();

                        if (errorCount > 0)
                        {
                            break;
                        }

                        Thread.Sleep(500); // errors are updated in next tx when we update the stats
                    }

                    Assert.Equal(1, errorCount);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        using (context.OpenReadTransaction())
                        {
                            var indexStats = index.GetIndexingState(context);

                            Assert.True(indexStats.IsStale);
                        }
                    }

                    var indexingErrors = index.GetErrors();

                    Assert.Equal(1, indexingErrors.Count);
                }
            }
        }
Exemplo n.º 13
0
        public void StalenessCalculationShouldWorkForAllDocsIndexes()
        {
            using (var database = CreateDocumentDatabase())
            {
                using (var index = MapIndex.CreateNew(new IndexDefinition()
                {
                    Name = "Index1",
                    Maps = { "from doc in docs select new { doc.Name }" },
                }, database))
                {
                    using (var queryContext = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        var context = queryContext.Documents;

                        using (var tx = context.OpenWriteTransaction())
                        {
                            using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
                            {
                                ["Name"] = "John",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Users"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "users/1", null, doc);
                            }

                            using (var doc = CreateDocument(context, "people/1", new DynamicJsonValue
                            {
                                ["Name"] = "Edward",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "People"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "people/1", null, doc);
                            }

                            tx.Commit();
                        }

                        using (context.OpenReadTransaction())
                        {
                            var isStale = index.IsStale(queryContext);
                            Assert.True(isStale);
                        }

                        var batchStats = new IndexingRunStats();
                        var scope      = new IndexingStatsScope(batchStats);
                        index.DoIndexingWork(scope, CancellationToken.None);

                        Assert.Equal(2, batchStats.MapAttempts);
                        Assert.Equal(2, batchStats.MapSuccesses);
                        Assert.Equal(0, batchStats.MapErrors);

                        using (context.OpenReadTransaction())
                        {
                            var isStale = index.IsStale(queryContext);
                            Assert.False(isStale);
                        }

                        using (var tx = context.OpenWriteTransaction())
                        {
                            database.DocumentsStorage.Delete(context, "people/1", null);

                            tx.Commit();
                        }

                        using (context.OpenReadTransaction())
                        {
                            var isStale = index.IsStale(queryContext);
                            Assert.True(isStale);
                        }

                        batchStats = new IndexingRunStats();
                        scope      = new IndexingStatsScope(batchStats);
                        index.DoIndexingWork(scope, CancellationToken.None);

                        using (context.OpenReadTransaction())
                        {
                            var isStale = index.IsStale(queryContext);
                            Assert.False(isStale);
                        }
                    }
                }
            }
        }
Exemplo n.º 14
0
        public async Task The_easiest_static_index()
        {
            using (var database = CreateDocumentDatabase())
            {
                using (var index = MapIndex.CreateNew(new IndexDefinition()
                {
                    Name = "Users_ByName",
                    Maps = { "from user in docs.Users select new { user.Name }" },
                }, database))
                {
                    DocumentQueryResult queryResult;
                    using (var queryContext = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        var context = queryContext.Documents;

                        using (var tx = context.OpenWriteTransaction())
                        {
                            using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
                            {
                                ["Name"] = "John",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Users"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "users/1", null, doc);
                            }

                            using (var doc = CreateDocument(context, "users/2", new DynamicJsonValue
                            {
                                ["Name"] = "Edward",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Users"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "users/2", null, doc);
                            }

                            tx.Commit();
                        }

                        var batchStats = new IndexingRunStats();
                        var scope      = new IndexingStatsScope(batchStats);
                        index.DoIndexingWork(scope, CancellationToken.None);

                        Assert.Equal(2, batchStats.MapAttempts);
                        Assert.Equal(2, batchStats.MapSuccesses);
                        Assert.Equal(0, batchStats.MapErrors);

                        queryResult =
                            await index.Query(new IndexQueryServerSide($"FROM '{index.Name}'"), queryContext, OperationCancelToken.None);

                        Assert.Equal(2, queryResult.Results.Count);
                    }

                    using (var context = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        queryResult = await index.Query(new IndexQueryServerSide($"FROM '{index.Name}' WHERE Name = 'John'"), context, OperationCancelToken.None);

                        Assert.Equal(1, queryResult.Results.Count);
                        Assert.Equal("users/1", queryResult.Results[0].Id);
                    }
                }
            }
        }
Exemplo n.º 15
0
        public void NumberOfDocumentsAndTombstonesToProcessShouldBeCalculatedCorrectly()
        {
            using (var database = CreateDocumentDatabase())
            {
                using (var index = MapIndex.CreateNew(new IndexDefinition()
                {
                    Name = "Index1",
                    Maps = { "from doc in docs.Users select new { doc.Name }" },
                }, database))
                {
                    using (var queryContext = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        var context = queryContext.Documents;

                        using (var tx = context.OpenWriteTransaction())
                        {
                            using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
                            {
                                ["Name"] = "John",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Users"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "users/1", null, doc);
                            }

                            using (var doc = CreateDocument(context, "users/2", new DynamicJsonValue
                            {
                                ["Name"] = "Bob",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Users"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "users/2", null, doc);
                            }

                            using (var doc = CreateDocument(context, "people/1", new DynamicJsonValue
                            {
                                ["Name"] = "Edward",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "People"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "people/1", null, doc);
                            }

                            tx.Commit();
                        }

                        IndexProgress progress;
                        using (context.OpenReadTransaction())
                        {
                            progress = index.GetProgress(queryContext);
                        }

                        Assert.Equal(0, progress.Collections["Users"].LastProcessedDocumentEtag);
                        Assert.Equal(0, progress.Collections["Users"].LastProcessedTombstoneEtag);
                        Assert.Equal(2, progress.Collections["Users"].NumberOfDocumentsToProcess);
                        Assert.Equal(0, progress.Collections["Users"].NumberOfTombstonesToProcess);
                        Assert.Equal(2, progress.Collections["Users"].TotalNumberOfDocuments);
                        Assert.Equal(0, progress.Collections["Users"].TotalNumberOfTombstones);

                        var batchStats = new IndexingRunStats();
                        var scope      = new IndexingStatsScope(batchStats);
                        index.DoIndexingWork(scope, CancellationToken.None);

                        using (context.OpenReadTransaction())
                        {
                            progress = index.GetProgress(queryContext);
                        }

                        Assert.Equal(2, progress.Collections["Users"].LastProcessedDocumentEtag);
                        Assert.Equal(0, progress.Collections["Users"].LastProcessedTombstoneEtag);
                        Assert.Equal(0, progress.Collections["Users"].NumberOfDocumentsToProcess);
                        Assert.Equal(0, progress.Collections["Users"].NumberOfTombstonesToProcess);
                        Assert.Equal(2, progress.Collections["Users"].TotalNumberOfDocuments);
                        Assert.Equal(0, progress.Collections["Users"].TotalNumberOfTombstones);

                        using (var tx = context.OpenWriteTransaction())
                        {
                            database.DocumentsStorage.Delete(context, "users/1", null);

                            using (var doc = CreateDocument(context, "users/3", new DynamicJsonValue
                            {
                                ["Name"] = "George",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Users"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "users/3", null, doc);
                            }

                            using (var doc = CreateDocument(context, "people/2", new DynamicJsonValue
                            {
                                ["Name"] = "Edward",
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "People"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "people/2", null, doc);
                            }

                            tx.Commit();
                        }

                        using (context.OpenReadTransaction())
                        {
                            progress = index.GetProgress(queryContext);
                        }

                        Assert.Equal(2, progress.Collections["Users"].LastProcessedDocumentEtag);
                        Assert.Equal(0, progress.Collections["Users"].LastProcessedTombstoneEtag);
                        Assert.Equal(1, progress.Collections["Users"].NumberOfDocumentsToProcess);
                        Assert.Equal(1, progress.Collections["Users"].NumberOfTombstonesToProcess);
                        Assert.Equal(2, progress.Collections["Users"].TotalNumberOfDocuments);
                        Assert.Equal(1, progress.Collections["Users"].TotalNumberOfTombstones);

                        batchStats = new IndexingRunStats();
                        scope      = new IndexingStatsScope(batchStats);
                        index.DoIndexingWork(scope, CancellationToken.None);

                        using (context.OpenReadTransaction())
                        {
                            progress = index.GetProgress(queryContext);
                        }

                        Assert.Equal(5, progress.Collections["Users"].LastProcessedDocumentEtag);
                        Assert.Equal(4, progress.Collections["Users"].LastProcessedTombstoneEtag);
                        Assert.Equal(0, progress.Collections["Users"].NumberOfDocumentsToProcess);
                        Assert.Equal(0, progress.Collections["Users"].NumberOfTombstonesToProcess);
                        Assert.Equal(2, progress.Collections["Users"].TotalNumberOfDocuments);
                        Assert.Equal(1, progress.Collections["Users"].TotalNumberOfTombstones);
                    }
                }
            }
        }
Exemplo n.º 16
0
        public async Task Static_map_reduce_index_with_multiple_outputs_per_document()
        {
            using (var database = CreateDocumentDatabase())
            {
                using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
                {
                    Name = "Users_ByCount_GroupByLocation",
                    Maps = { @"from order in docs.Orders
from line in order.Lines
select new { Product = line.Product, Count = 1, Total = line.Price }" },
                    Reduce = @"from result in mapResults
group result by result.Product into g
select new
{
    Product = g.Key,
    Count = g.Sum(x=> x.Count),
    Total = g.Sum(x=> x.Total)
}",
                    Fields =
                    {
                        { "Product", new IndexFieldOptions {
                              Storage = FieldStorage.Yes
                          } }
                    }
                }, database))
                {
                    DocumentQueryResult queryResult;
                    using (var queryContext = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        var context = queryContext.Documents;

                        using (var tx = context.OpenWriteTransaction())
                        {
                            using (var doc = CreateDocument(context, "orders/1", new DynamicJsonValue
                            {
                                ["Lines"] = new DynamicJsonArray
                                {
                                    new DynamicJsonValue
                                    {
                                        ["Product"] = "Milk",
                                        ["Price"] = 10.5
                                    },
                                    new DynamicJsonValue
                                    {
                                        ["Product"] = "Bread",
                                        ["Price"] = 10.7
                                    }
                                },
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Orders"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "orders/1", null, doc);
                            }

                            using (var doc = CreateDocument(context, "orders/2", new DynamicJsonValue
                            {
                                ["Lines"] = new DynamicJsonArray
                                {
                                    new DynamicJsonValue
                                    {
                                        ["Product"] = "Milk",
                                        ["Price"] = 10.5
                                    }
                                },
                                [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                {
                                    [Constants.Documents.Metadata.Collection] = "Orders"
                                }
                            }))
                            {
                                database.DocumentsStorage.Put(context, "orders/2", null, doc);
                            }

                            tx.Commit();
                        }

                        var batchStats = new IndexingRunStats();
                        var scope      = new IndexingStatsScope(batchStats);
                        while (index.DoIndexingWork(scope, CancellationToken.None))
                        {
                        }

                        queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), queryContext, OperationCancelToken.None);

                        Assert.Equal(2, queryResult.Results.Count);
                    }
                    using (var context = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Product = 'Milk'"), context, OperationCancelToken.None);

                        Assert.Equal(1, queryResult.Results.Count);
                        Assert.Equal("Milk", queryResult.Results[0].Data["Product"].ToString());
                        Assert.Equal(2L, queryResult.Results[0].Data["Count"]);
                        Assert.Equal(21.0, (LazyNumberValue)queryResult.Results[0].Data["Total"]);
                    }
                }
            }
        }
Exemplo n.º 17
0
        public void CanSetMapBatchSize(int?mapBatchSize, int numberOfDocs)
        {
            var indexDefinition = new IndexDefinition
            {
                Name = "NewIndex",
                Maps = new HashSet <string>
                {
                    "from p in docs.Orders select new { CompanyName = LoadDocument(p.Company, \"Companies\").Name }"
                },
                Configuration = new IndexConfiguration
                {
                    {
                        RavenConfiguration.GetKey(x => x.Indexing.MapBatchSize), mapBatchSize?.ToString()
                    }
                }
            };

            using (var database = CreateDocumentDatabase())
                using (var index = MapIndex.CreateNew(indexDefinition, database))
                    using (var queryContext = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        var context = queryContext.Documents;

                        Assert.Equal(mapBatchSize, index.Configuration.MapBatchSize);

                        using (var tx = context.OpenWriteTransaction())
                        {
                            for (var i = 0; i < numberOfDocs; i++)
                            {
                                var orderDocumentId   = $"orders/{i}";
                                var companyDocumentId = $"companies/{i}";
                                using (var doc = CreateDocument(context, orderDocumentId, new DynamicJsonValue
                                {
                                    ["Name"] = "John",
                                    ["Company"] = companyDocumentId,
                                    [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                    {
                                        [Constants.Documents.Metadata.Collection] = "Orders"
                                    }
                                }))
                                {
                                    database.DocumentsStorage.Put(context, orderDocumentId, null, doc);
                                }
                            }

                            tx.Commit();
                        }

                        using (var tx = context.OpenWriteTransaction())
                        {
                            for (var i = 0; i < numberOfDocs; i++)
                            {
                                var companyDocumentId = $"companies/{i}";
                                using (var doc = CreateDocument(context, companyDocumentId, new DynamicJsonValue
                                {
                                    ["Name"] = "RavenDB",
                                    [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                    {
                                        [Constants.Documents.Metadata.Collection] = "Companies"
                                    }
                                }))
                                {
                                    database.DocumentsStorage.Put(context, companyDocumentId, null, doc);
                                }
                            }

                            tx.Commit();
                        }

                        var numberOfBatches = numberOfDocs / mapBatchSize;
                        var batchStats      = new IndexingRunStats();
                        var stats           = new IndexingStatsScope(batchStats);

                        for (var i = 0; i < numberOfBatches; i++)
                        {
                            index.DoIndexingWork(stats, CancellationToken.None);
                            Assert.Equal((i + 1) * mapBatchSize, stats.MapAttempts);
                        }

                        index.DoIndexingWork(stats, CancellationToken.None);
                        Assert.Equal(numberOfDocs, stats.MapAttempts);

                        using (var tx = context.OpenWriteTransaction())
                        {
                            for (var i = 0; i < numberOfDocs; i++)
                            {
                                var companyDocumentId = $"companies/{i}";
                                using (var doc = CreateDocument(context, companyDocumentId, new DynamicJsonValue
                                {
                                    ["Name"] = "Hibernating Rhinos",
                                    [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                    {
                                        [Constants.Documents.Metadata.Collection] = "Companies"
                                    }
                                }))
                                {
                                    database.DocumentsStorage.Put(context, companyDocumentId, null, doc);
                                }
                            }


                            tx.Commit();
                        }

                        batchStats = new IndexingRunStats();
                        stats      = new IndexingStatsScope(batchStats);

                        for (var i = 0; i < numberOfBatches; i++)
                        {
                            index.DoIndexingWork(stats, CancellationToken.None);
                            Assert.Equal((i + 1) * mapBatchSize, stats.MapReferenceAttempts);
                        }

                        index.DoIndexingWork(stats, CancellationToken.None);
                        Assert.Equal(numberOfDocs, stats.MapReferenceAttempts);

                        using (context.OpenReadTransaction())
                            Assert.False(index.IsStale(queryContext));
                    }
        }
Exemplo n.º 18
0
        public async Task GroupByMultipleFields()
        {
            using (var db = CreateDocumentDatabase())
                using (var index = AutoMapReduceIndex.CreateNew(new AutoMapReduceIndexDefinition("Orders", new[]
                {
                    new AutoIndexField
                    {
                        Name = "Count",
                        Aggregation = AggregationOperation.Count,
                        Storage = FieldStorage.Yes
                    }
                }, new[]
                {
                    new AutoIndexField
                    {
                        Name = "Employee",
                        Storage = FieldStorage.Yes
                    },
                    new AutoIndexField
                    {
                        Name = "Company",
                        Storage = FieldStorage.Yes
                    },
                }), db))
                {
                    CreateOrders(db, 10, employees: new[] { "employees/1", "employees/2" }, companies: new[] { "companies/1", "companies/2", "companies/3" });

                    index.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var results = (await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None)).Results;

                        context.CloseTransaction();

                        Assert.Equal(6, results.Count);
                    }
                    for (int i = 0; i < 6; i++)
                    {
                        using (var context = QueryOperationContext.ShortTermSingleUse(db))
                        {
                            var employeeNumber = i % 2 + 1;
                            var companyNumber  = i % 3 + 1;
                            var results        = (await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}' WHERE Employee = 'employees/{employeeNumber}' AND Company = 'companies/{companyNumber}'")
                            {
                                Query = $"Employee:employees/{employeeNumber} AND Company:companies/{companyNumber}"
                            }, context, OperationCancelToken.None)).Results;

                            Assert.Equal(1, results.Count);

                            long expectedCount;

                            if ((employeeNumber == 1 && companyNumber == 2) || (employeeNumber == 2 && companyNumber == 3))
                            {
                                expectedCount = 1;
                            }
                            else
                            {
                                expectedCount = 2;
                            }

                            Assert.Equal(expectedCount, results[0].Data["Count"]);
                        }
                    }
                }
        }
Exemplo n.º 19
0
        public async Task Reduction_should_ignore_overflow_pages(long numberOfDocs)
        {
            using (var database = CreateDocumentDatabase())
            {
                using (var index = MapReduceIndex.CreateNew <MapReduceIndex>(new IndexDefinition()
                {
                    Name = "Users_ByLocation",
                    Maps = { @"from user in docs.Users
select new { Location = user.Location, Count = 1 }" },
                    Reduce = @"from result in results
group result by result.Location into g
select new
{
    Location = g.Key,
    Count = g.Sum(x=> x.Count)
}",
                }, database))
                {
                    using (var queryContext = QueryOperationContext.ShortTermSingleUse(database))
                    {
                        var context = queryContext.Documents;

                        var bytes = new byte[4096];
                        new Random(2).NextBytes(bytes);

                        var randomLocation = Encoding.ASCII.GetString(bytes);

                        using (var tx = context.OpenWriteTransaction())
                        {
                            for (int i = 0; i < numberOfDocs; i++)
                            {
                                var user = new DynamicJsonValue()
                                {
                                    ["Location"] = randomLocation,
                                    [Constants.Documents.Metadata.Key] = new DynamicJsonValue
                                    {
                                        [Constants.Documents.Metadata.Collection] = "Users"
                                    }
                                };

                                using (var doc = CreateDocument(context, $"users/{i}", user))
                                {
                                    database.DocumentsStorage.Put(context, $"users/{i}", null, doc);
                                }
                            }

                            tx.Commit();
                        }

                        var firstRunStats = new IndexingRunStats();
                        var scope         = new IndexingStatsScope(firstRunStats);
                        index.DoIndexingWork(scope, CancellationToken.None);

                        Assert.Equal(numberOfDocs, firstRunStats.MapAttempts);
                        Assert.Equal(numberOfDocs, firstRunStats.MapSuccesses);
                        Assert.Equal(0, firstRunStats.MapErrors);

                        var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), queryContext, OperationCancelToken.None);

                        Assert.False(queryResult.IsStale);

                        Assert.Equal(1, queryResult.Results.Count);
                        Assert.Equal(numberOfDocs, queryResult.Results[0].Data["Count"]);
                    }
                }
            }
        }
        public async Task MultipleAggregationFunctionsCanBeUsed()
        {
            using (var db = CreateDocumentDatabase())
                using (var mri = AutoMapReduceIndex.CreateNew(new AutoMapReduceIndexDefinition("Users", new[]
                {
                    new AutoIndexField
                    {
                        Name = "Count",
                        Aggregation = AggregationOperation.Count,
                        Storage = FieldStorage.Yes
                    },
                    new AutoIndexField
                    {
                        Name = "TotalCount",
                        Aggregation = AggregationOperation.Count,
                        Storage = FieldStorage.Yes
                    },
                    new AutoIndexField
                    {
                        Name = "Age",
                        Aggregation = AggregationOperation.Sum,
                        Storage = FieldStorage.Yes
                    }
                }, new[]
                {
                    new AutoIndexField
                    {
                        Name = "Location",
                        Storage = FieldStorage.Yes
                    },
                }), db))
                {
                    BasicAutoMapReduceIndexing.CreateUsers(db, 2, "Poland");

                    mri.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await mri.Query(new IndexQueryServerSide($"FROM INDEX '{mri.Name}'"), context, OperationCancelToken.None);

                        Assert.Equal(1, queryResult.Results.Count);
                        var result = queryResult.Results[0].Data;

                        string location;
                        Assert.True(result.TryGet("Location", out location));
                        Assert.Equal("Poland", location);

                        Assert.Equal(2L, result["Count"]);

                        Assert.Equal(2L, result["TotalCount"]);

                        Assert.Equal(41L, result["Age"]);
                    }

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await mri.Query(new IndexQueryServerSide($"FROM INDEX '{mri.Name}' WHERE Count BETWEEN 2 AND 10"), context,
                                                          OperationCancelToken.None);

                        Assert.Equal(1, queryResult.Results.Count);
                    }

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await mri.Query(new IndexQueryServerSide($"FROM INDEX '{mri.Name}' WHERE Count >= 10"), context, OperationCancelToken.None);

                        Assert.Equal(0, queryResult.Results.Count);
                    }
                }
        }
Exemplo n.º 21
0
        public async Task CanDelete()
        {
            const long numberOfUsers = 10;

            using (var db = CreateDocumentDatabase())
                using (var index = AutoMapReduceIndex.CreateNew(GetUsersCountByLocationIndexDefinition(), db))
                {
                    CreateUsers(db, numberOfUsers, "Poland");

                    // index 10 users
                    index.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);

                        var results = queryResult.Results;

                        Assert.Equal(1, results.Count);

                        Assert.Equal("Poland", results[0].Data["Location"].ToString());
                        Assert.Equal(numberOfUsers, results[0].Data["Count"]);
                    }

                    using (var context = DocumentsOperationContext.ShortTermSingleUse(db))
                    {
                        using (var tx = context.OpenWriteTransaction())
                        {
                            db.DocumentsStorage.Delete(context, "users/0", null);

                            tx.Commit();
                        }
                    }

                    // one document deleted
                    index.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);

                        var results = queryResult.Results;

                        Assert.Equal(1, results.Count);

                        Assert.Equal("Poland", results[0].Data["Location"].ToString());
                        Assert.Equal(numberOfUsers - 1, results[0].Data["Count"]);
                    }

                    CreateUsers(db, 1, "Poland");

                    // document added again
                    index.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);

                        var results = queryResult.Results;

                        Assert.Equal(1, results.Count);

                        Assert.Equal("Poland", results[0].Data["Location"].ToString());
                        Assert.Equal(numberOfUsers, results[0].Data["Count"]);
                    }

                    using (var context = DocumentsOperationContext.ShortTermSingleUse(db))
                    {
                        using (var tx = context.OpenWriteTransaction())
                        {
                            for (int i = 0; i < numberOfUsers; i++)
                            {
                                db.DocumentsStorage.Delete(context, $"users/{i}", null);
                            }

                            tx.Commit();
                        }
                    }

                    // all documents removed
                    index.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);

                        var results = queryResult.Results;

                        Assert.Equal(0, results.Count);
                    }

                    CreateUsers(db, numberOfUsers, "Poland");

                    // documents added back
                    index.DoIndexingWork(new IndexingStatsScope(new IndexingRunStats()), CancellationToken.None);

                    using (var context = QueryOperationContext.ShortTermSingleUse(db))
                    {
                        var queryResult = await index.Query(new IndexQueryServerSide($"FROM INDEX '{index.Name}'"), context, OperationCancelToken.None);

                        var results = queryResult.Results;

                        Assert.Equal(1, results.Count);

                        Assert.Equal("Poland", results[0].Data["Location"].ToString());
                        Assert.Equal(numberOfUsers, results[0].Data["Count"]);
                    }
                }
        }