예제 #1
0
        public async IAsyncEnumerable <IndexKey <TKeyType> > ReadIndexKeys <TDataType, TKeyType>(Expression <Func <TDataType, TKeyType> > indexProperty) where TDataType : DataEntity
        {
            var indexMember = ((MemberExpression)indexProperty.Body).Member;

            using (Logger.BeginScope("{Operation} is {Action} {IndexName} for {DataType}",
                                     nameof(DataManager), "getting keys from index", indexMember.Name, typeof(TDataType).Name))
            {
                if (indexMember.IsDefined(typeof(IndexAttribute), true))
                {
                    Logger.LogTrace("Property is indexed");
                    var indexName = indexMember.Name;
                    if (await IndexWorker.IndexExist <TDataType>(indexName))
                    {
                        var keys = IndexWorker.GetKeysFromIndex <TDataType, TKeyType>(indexName);
                        Logger.LogInformation("Found keys in index", keys);
                        await foreach (var key in keys)
                        {
                            yield return(key);
                        }
                    }

                    yield break;
                }

                Logger.LogWarning("The property used is not marked as indexed");
            }
        }
예제 #2
0
        public async IAsyncEnumerable <TDataType> ReadIndex <TDataType, TKeyType>(TKeyType key,
                                                                                  Expression <Func <TDataType, TKeyType> > indexProperty) where TDataType : DataEntity
        {
            var indexMember = ((MemberExpression)indexProperty.Body).Member;

            using (Logger.BeginScope("{Operation} is {Action} {IndexName} for {DataType} with a key {key}",
                                     nameof(DataManager), "read from index", indexMember.Name, typeof(TDataType).Name, key))
            {
                if (indexMember.IsDefined(typeof(IndexAttribute), true))
                {
                    Logger.LogTrace("Property is indexed");
                    var indexName = indexMember.Name;
                    if (await IndexWorker.IndexExist <TDataType>(indexName))
                    {
                        Logger.LogTrace("Index exists");
                        var ids = IndexWorker.GetIdsFromIndex <TDataType, TKeyType>(indexName, key);
                        Logger.LogInformation("Found ids in index", ids, key);
                        await foreach (var id in ids)
                        {
                            yield return(await Read <TDataType>(id));
                        }
                    }

                    yield break;
                }

                Logger.LogWarning("The property used is not marked as indexed");
            }
        }
예제 #3
0
        public IActionResult IndexAllShops(bool clean = true)
        {
            var context = new IndexAllShopsContext();
            var worker  = new IndexWorker(_settings, context, _works, _dbHelper);

            return(_works.AddToQueue(worker.IndexAll, context, QueuePriority.Parallel, clean));
        }
예제 #4
0
        public IndexWorkerFixture()
        {
            _mockConfig = _mockRepo.Create <ITargetIndexingConfig>();

            _mockEsClient = _mockRepo.Create <IEsIndexClient>();

            _testWorker = new IndexWorker(_mockConfig.Object, _mockEsClient.Object);
        }
 protected override Action GetWorker(List <EsDocument> data)
 {
     return(new Action(() =>
     {
         using (var workerWrapper = _workerFactory.CreateReleasable((f) => f.Create()))
         {
             IndexWorker worker = workerWrapper;
             worker.Index(data);
         }
     }));
 }
예제 #6
0
        public IActionResult IndexShop(
            int id,
            bool downloadFresh = false,
            bool needLink      = true,
            bool needSoldOut   = true,
            bool clean         = true)
        {
            var context = new IndexShopContext(id, downloadFresh, needLink, needSoldOut);
            var worker  = new IndexWorker(_settings, context, _works, _dbHelper);

            return(_works.AddToQueue(worker.Index, context, QueuePriority.Parallel, clean));
        }
예제 #7
0
        public void BulkIndex_addsMultipleDoc_Successfully(
            EsDocument doc1,
            EsDocument doc2,
            EsDocument doc3,
            EsTestIndexClient testClient,
            IndexWorker indexWorker)
        {
            indexWorker
            .Index(new[] { doc1, doc2, doc3 })
            .Should().Be(true);

            using (testClient.ForTestAssertions())
            {
                testClient.GetAllDocs().Should().HaveCount(3);
            }
        }
예제 #8
0
        public void BulkIndex_addsSingleDoc_Successfully(
            EsDocument doc1,
            EsTestIndexClient testClient,
            IndexWorker indexWorker)
        {
            indexWorker
            .Index(new List <EsDocument>()
            {
                doc1
            })
            .Should().Be(true);

            using (testClient.ForTestAssertions())
            {
                testClient.GetAllDocs().Single().ToString().Should().Be(doc1.Data.ToString());
            }
        }
        public void ScrollPage_MultipleDocsExistOnSingleScrollPage_ReturnsDocs(
            EsDocument doc1,
            EsDocument doc2,
            EsDocument doc3,
            IndexWorker indexWorker,
            EsTestIndexClient testClient,
            ScrollWorker scrollWorker)
        {
            indexWorker.Index(new [] { doc1, doc2, doc3 });

            using (testClient.ForTestAssertions())
            {
                var actuals = scrollWorker.ScrollPage();

                actuals.Should().NotBeNull();
                actuals.Should().HaveCount(3);
            }
        }
        public void ScrollPage_MultipleDocsExistOnMultipleScrollPages_ReturnsDocs(
            IFixture fixture,
            IndexWorker indexWorker,
            EsTestIndexClient testClient,
            ScrollWorker scrollWorker)
        {
            var testDocs = fixture.CreateMany <EsDocument>(scrollWorker.Client.Config.BatchSize * 2);

            indexWorker.Index(testDocs);

            using (testClient.ForTestAssertions())
            {
                var actuals = scrollWorker.ScrollPage();             // first page
                actuals = actuals.Concat(scrollWorker.ScrollPage()); // second page
                scrollWorker.ScrollPage().Should().BeEmpty();        // third page is empty

                actuals.Should().NotBeNull();
                actuals.Should().HaveSameCount(testDocs);
            }
        }
예제 #11
0
        public async Task <bool> Restore <TDataType>(int id) where TDataType : DataEntity
        {
            using (Logger.BeginScope("{Operation} is {Action} {DataType} with Id ({Id})", nameof(DataManager),
                                     "restoring", typeof(TDataType).Name, id))
            {
                var data = await Read <TDataType>(id);

                if (data is null)
                {
                    Logger.LogTrace("Data could be deleted");
                    if (await DataStreamer.Undelete <TDataType>(id))
                    {
                        Logger.LogTrace("Data restored to store");
                        data = await Read <TDataType>(id);

                        if (typeof(TDataType).IsDefined(typeof(AuditAttribute), true))
                        {
                            Logger.LogTrace("Data is audited");
                            if (await AuditWorker.UndeleteEvent(data))
                            {
                                if (await AuditWorker.CommitEvents(data))
                                {
                                    Logger.LogTrace("Audit committed to store");
                                    if (await IndexWorker.Index(data))
                                    {
                                        Logger.LogInformation("Restored data with audit");
                                        return(true);
                                    }

                                    Logger.LogWarning("Failed to index data");
                                    await AuditWorker.RollbackEvent(data);
                                }
                                else
                                {
                                    Logger.LogWarning("Failed to commit audit for data to store");
                                    await AuditWorker.DiscardEvents(data);

                                    Logger.LogTrace("Discarded audit record");
                                }
                            }
                            else
                            {
                                Logger.LogWarning("Failed to create undelete audit record for data");
                            }
                        }
                        else
                        {
                            if (await IndexWorker.Index(data))
                            {
                                Logger.LogInformation("Restored data");
                                return(true);
                            }

                            Logger.LogWarning("Failed to index data");
                        }

                        await DataStreamer.Delete <TDataType>(id);

                        Logger.LogInformation("Rolled back restore");
                    }

                    Logger.LogWarning("Failed to restore data");
                    return(false);
                }

                Logger.LogWarning("Data didn't need restoring");
                return(true);
            }
        }
예제 #12
0
        public async Task <bool> Delete <TDataType>(int id) where TDataType : DataEntity
        {
            using (Logger.BeginScope("{Operation} is {Action} {DataType} with Id ({Id})", nameof(DataManager),
                                     "deleting", typeof(TDataType).Name, id))
            {
                var data = await Read <TDataType>(id);

                if (data is null)
                {
                    Logger.LogWarning("Unable to find the data it may already be deleted");
                    return(false);
                }

                if (typeof(TDataType).IsDefined(typeof(AuditAttribute), true))
                {
                    Logger.LogTrace("Data is audited");
                    if (await AuditWorker.DeleteEvent(data))
                    {
                        Logger.LogTrace("Delete audit record created");
                        if (await DataStreamer.Delete <TDataType>(id))
                        {
                            Logger.LogTrace("Data deleted from store");
                            if (await AuditWorker.CommitEvents(data))
                            {
                                Logger.LogTrace("Audit committed to store");
                                if (await IndexWorker.Unindex(data))
                                {
                                    Logger.LogInformation("Data deleted from store");
                                    return(true);
                                }

                                Logger.LogWarning("Failed to remove indexes for data");
                                await AuditWorker.RollbackEvent(data);

                                Logger.LogInformation("Added rollback audit record");
                            }
                            else
                            {
                                Logger.LogWarning("Failed to commit audit to store");
                            }

                            await DataStreamer.Rollback(id, data);

                            Logger.LogInformation("Rolled data back");
                        }
                        else
                        {
                            Logger.LogWarning("Failed to delete data");
                            await AuditWorker.DiscardEvents(data);

                            Logger.LogTrace("Discarded audit record");
                        }

                        return(false);
                    }

                    Logger.LogWarning("Failed to create audit record for delete");
                    return(false);
                }

                Logger.LogTrace("Data is not audited");
                if (await DataStreamer.Delete <TDataType>(id))
                {
                    Logger.LogInformation("Deleted data");
                    if (await IndexWorker.Unindex(data))
                    {
                        Logger.LogInformation("Removed any indexes for data");
                        return(true);
                    }

                    Logger.LogWarning("Failed to remove indexes for data");
                    await DataStreamer.Rollback(id, data);

                    Logger.LogInformation("Rolled data back");
                }

                Logger.LogWarning("Failed to delete data");
                return(false);
            }
        }
예제 #13
0
        public async Task <bool> Write <TDataType>(TDataType data) where TDataType : DataEntity
        {
            using (Logger.BeginScope("{Operation} is {Action} {DataType} with Id ({Id})", nameof(DataManager),
                                     "writing", typeof(TDataType).Name, data.Id))
            {
                Logger.LogTrace("Checking if data exists");
                var currentData = await Read <TDataType>(data.Id);

                Logger.LogInformation(currentData is null ? "Creating data" : "Updating data");
                if (typeof(TDataType).IsDefined(typeof(AuditAttribute), true))
                {
                    Logger.LogTrace("{DataType} is audited", typeof(TDataType).Name);
                    var writtenData = await DataStreamer.Write(data.Id, data);

                    if (writtenData)
                    {
                        Logger.LogTrace("Data ready to commit to store");
                        var auditResult = currentData is null
                            ? await AuditWorker.CreationEvent(data)
                            : await AuditWorker.UpdateEvent(data, currentData);

                        if (auditResult)
                        {
                            Logger.LogTrace("Audit record created");
                            writtenData = await DataStreamer.Commit <TDataType>(data.Id);

                            if (writtenData)
                            {
                                Logger.LogTrace("Data saved to store");
                                var committedAudit = await AuditWorker.CommitEvents(data);

                                if (committedAudit)
                                {
                                    Logger.LogTrace("Audit saved to store");
                                    var indexed = await IndexWorker.Index(data);

                                    if (indexed)
                                    {
                                        Logger.LogInformation("Data, Audit saved successfully");
                                        return(true);
                                    }

                                    Logger.LogWarning("Failed to index data");
                                    await AuditWorker.RollbackEvent(data);

                                    Logger.LogInformation("Added rollback to audit");
                                    Logger.LogTrace("Rolled back data");
                                }

                                await DataStreamer.Rollback(data.Id, currentData);

                                Logger.LogInformation("Rolled data back");
                                return(false);
                            }

                            await AuditWorker.DiscardEvents(data);

                            return(false);
                        }

                        Logger.LogWarning("Failed to create audit record for data");
                        await DataStreamer.DiscardChanges <TDataType>(data.Id);

                        Logger.LogTrace("Discard changes");
                    }

                    Logger.LogWarning("Failed to save data to the store");
                    return(false);
                }

                Logger.LogTrace("Data is not audited");
                var writeAndCommit = await DataStreamer.WriteAndCommit(data.Id, data);

                if (writeAndCommit)
                {
                    var indexed = await IndexWorker.Index(data);

                    if (!indexed)
                    {
                        Logger.LogWarning("Failed to index data");
                        await DataStreamer.Rollback(data.Id, currentData);

                        Logger.LogInformation("Rolled data back");
                        return(false);
                    }

                    Logger.LogTrace("Data was indexed");
                    Logger.LogInformation("Data saved successfully");
                }
                else
                {
                    Logger.LogWarning("Failed to save data");
                }

                return(writeAndCommit);
            }
        }