Beispiel #1
0
        public AzureTablesConfigurationRepoFixture()
        {
            var    config = TestConfiguration.LoadConfig();
            string azureStorageConnectionString = config["AzureStorageConnectionString"];

            CloudTableSetup = new EventProcessingCloudTableSetup(azureStorageConnectionString, "test");
            ConfigRepo      = new AzureEventProcessingConfigurationRepository(CloudTableSetup);
        }
Beispiel #2
0
 public EventSubscriptionFactory(
     IEventProcessingConfigurationRepository configurationRepository,
     IEventMatcherFactory eventMatcherFactory,
     IEventHandlerFactory eventHandlerFactory)
 {
     ConfigurationRepository = configurationRepository;
     EventMatcherFactory     = eventMatcherFactory;
     EventHandlerFactory     = eventHandlerFactory;
 }
Beispiel #3
0
        public EventSubscriptionFactory(
            IWeb3 web3,
            IEventProcessingConfigurationRepository configurationRepository,
            ISubscriberQueueFactory subscriberQueueFactory             = null,
            ISubscriberSearchIndexFactory subscriberSearchIndexFactory = null,
            ISubscriberStorageFactory subscriberRepositoryFactory      = null) : this(
                configurationRepository,

                new EventMatcherFactory(
                    configurationRepository.ParameterConditions,
                    configurationRepository.EventSubscriptionAddresses,
                    configurationRepository.SubscriberContracts),

                new EventHandlerFactory(
                    web3,
                    configurationRepository,
                    subscriberQueueFactory,
                    subscriberSearchIndexFactory,
                    subscriberRepositoryFactory))
        {
        }
Beispiel #4
0
 public EventHandlerFactory(
     IWeb3 web3,
     IEventProcessingConfigurationRepository configRepo,
     ISubscriberQueueFactory subscriberQueueFactory             = null,
     ISubscriberSearchIndexFactory subscriberSearchIndexFactory = null,
     ISubscriberStorageFactory subscriberRepositoryFactory      = null)
     : this(
         web3.Eth,
         configRepo.EventSubscriptionStates,
         configRepo.EventContractQueries,
         configRepo.EventAggregators,
         web3.Eth.Transactions.GetTransactionByHash,
         configRepo.SubscriberQueues,
         subscriberQueueFactory,
         configRepo.SubscriberSearchIndexes,
         subscriberSearchIndexFactory,
         configRepo.EventRules,
         configRepo.SubscriberStorage,
         subscriberRepositoryFactory)
 {
 }
        public async Task WriteAnyMakerEventToQueue()
        {
            var    config = TestConfiguration.LoadConfig();
            string azureStorageConnectionString = config["AzureStorageConnectionString"];

            var configurationContext = MakerDAOEventProcessingConfig.Create(PARTITION, out IdGenerator idGenerator);
            IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator);

            var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet);

            // queue components
            var queueFactory = new AzureSubscriberQueueFactory(azureStorageConnectionString);

            // load subscribers and event subscriptions
            var eventSubscriptionFactory = new EventSubscriptionFactory(
                web3, configurationRepository, queueFactory);

            List <IEventSubscription> eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION);

            // progress repo (dictates which block ranges to process next)
            // maintain separate progress per partition via a prefix
            var storageCloudSetup = new CloudTableSetup(azureStorageConnectionString, prefix: $"Partition{PARTITION}");
            var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository();

            //this ensures we only query the chain for events relating to this contract
            var makerAddressFilter = new NewFilterInput()
            {
                Address = new[] { MakerDAOEventProcessingConfig.MAKER_CONTRACT_ADDRESS }
            };

            // load service
            var progressService       = new BlockProgressService(web3, MIN_BLOCK_NUMBER, blockProgressRepo);
            var logProcessor          = new BlockRangeLogsProcessor(web3, eventSubscriptions, makerAddressFilter);
            var batchProcessorService = new LogsProcessor(logProcessor, progressService, MAX_BLOCKS_PER_BATCH);

            // execute
            var blockRangesProcessed = new List <BlockRange?>();

            try
            {
                for (var i = 0; i < 2; i++)  // 2 batch iterations
                {
                    var ctx            = new System.Threading.CancellationTokenSource();
                    var rangeProcessed = await batchProcessorService.ProcessOnceAsync(ctx.Token);

                    blockRangesProcessed.Add(rangeProcessed);

                    // save event subscription state after each batch
                    await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State));
                }
            }
            finally
            {
                await ClearDown(configurationContext, storageCloudSetup, queueFactory);
            }

            var subscriptionState = await configurationRepository.EventSubscriptionStates.GetAsync(eventSubscriptions[0].Id);

            Assert.Equal(2, (int)subscriptionState.Values["HandlerInvocations"]);
            Assert.Equal(28, (int)subscriptionState.Values["EventsHandled"]);
        }
Beispiel #6
0
        public async Task WebJobExample()
        {
            var    config = TestConfiguration.LoadConfig();
            string azureStorageConnectionString = config["AzureStorageConnectionString"];
            string azureSearchKey = config["AzureSearchApiKey"];

            var configurationContext = EventProcessingConfigMock.Create(PARTITION, out IdGenerator idGenerator);
            IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator);

            var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby);

            // search components
            var searchService = new AzureSearchService(serviceName: AZURE_SEARCH_SERVICE_NAME, searchApiKey: azureSearchKey);

            var subscriberSearchIndexFactory = new SubscriberSearchIndexFactory(async indexName =>
            {
                if (await searchService.IndexExistsAsync(indexName) == false)
                {
                    //TODO: REPLACE THIS WITH Nethereum.BlockchainStore.Search.Azure.EventToGenericSearchDocMapper

                    await searchService.CreateIndexAsync(EventToGenericSearchDocMapper.CreateAzureIndexDefinition(indexName));
                }

                return(searchService.CreateIndexer <DecodedEvent, GenericSearchDocument>(
                           indexName, decodedEvent => EventToGenericSearchDocMapper.Map(decodedEvent, decodedEvent.State)));
            });

            // queue components
            //AzureStorageQueueFactory
            var azureQueueFactory = new AzureStorageQueueFactory(azureStorageConnectionString);

            var subscriberQueueFactory = new SubscriberQueueFactory(
                queueName => azureQueueFactory.GetOrCreateQueueAsync(queueName));

            // subscriber repository
            var repositoryFactory = new AzureTablesSubscriberRepositoryFactory(azureStorageConnectionString);

            // load subscribers and event subscriptions
            var eventSubscriptionFactory = new EventSubscriptionFactory(
                web3, configurationRepository, subscriberQueueFactory, subscriberSearchIndexFactory, repositoryFactory);

            var eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION);

            // progress repo (dictates which block ranges to process next)
            // maintain separate progress per partition via a prefix
            var storageCloudSetup = new AzureTablesRepositoryFactory(azureStorageConnectionString, prefix: $"Partition{PARTITION}");
            var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository();

            var logProcessor = web3.Processing.Logs.CreateProcessor(
                logProcessors: eventSubscriptions, blockProgressRepository: blockProgressRepo);

            // execute
            try
            {
                var ctx = new System.Threading.CancellationTokenSource();
                await logProcessor.ExecuteAsync(BLOCK_TO, ctx.Token, BLOCK_FROM);
            }
            finally
            {
                await ClearDown(configurationContext, storageCloudSetup, searchService, azureQueueFactory, repositoryFactory);
            }

            // save event subscription state
            await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State));

            // assertions

            var subscriptionState1 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 1); // interested in transfers with contract queries and aggregations
            var subscriptionState2 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 2); // interested in transfers with simple aggregation
            var subscriptionState3 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 3); // interested in any event for a specific address

            Assert.Equal("4009000000002040652615", subscriptionState1.Values["RunningTotalForTransferValue"].ToString());
            Assert.Equal((uint)19, subscriptionState2.Values["CurrentTransferCount"]);

            var txForSpecificAddress = (List <string>)subscriptionState3.Values["AllTransactionHashes"];

            Assert.Equal("0x362bcbc78a5cc6156e8d24d95bee6b8f53d7821083940434d2191feba477ae0e", txForSpecificAddress[0]);
            Assert.Equal("0xe63e9422dedf84d0ce13f9f75ebfd86333ce917b2572925fbdd51b51caf89b77", txForSpecificAddress[1]);

            var blockNumbersForSpecificAddress = (List <HexBigInteger>)subscriptionState3.Values["AllBlockNumbers"];

            Assert.Equal(4063362, blockNumbersForSpecificAddress[0].Value);
            Assert.Equal(4063362, blockNumbersForSpecificAddress[1].Value);
        }
        public async Task WebJobExample()
        {
            var    config = TestConfiguration.LoadConfig();
            string azureStorageConnectionString = config["AzureStorageConnectionString"];
            string azureSearchKey = config["AzureSearchApiKey"];

            var configurationContext = EventProcessingConfigMock.Create(PARTITION, out IdGenerator idGenerator);
            IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator);

            var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby);

            // search components
            var searchService      = new AzureSearchService(serviceName: AZURE_SEARCH_SERVICE_NAME, searchApiKey: azureSearchKey);
            var searchIndexFactory = new AzureSubscriberSearchIndexFactory(searchService);

            // queue components
            var queueFactory = new AzureSubscriberQueueFactory(azureStorageConnectionString);

            // subscriber repository
            var repositoryFactory = new AzureTablesSubscriberRepositoryFactory(azureStorageConnectionString);

            // load subscribers and event subscriptions
            var eventSubscriptionFactory = new EventSubscriptionFactory(
                web3, configurationRepository, queueFactory, searchIndexFactory, repositoryFactory);

            List <IEventSubscription> eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION);

            // progress repo (dictates which block ranges to process next)
            // maintain separate progress per partition via a prefix
            var storageCloudSetup = new CloudTableSetup(azureStorageConnectionString, prefix: $"Partition{PARTITION}");
            var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository();

            // load service
            var progressService       = new BlockProgressService(web3, MIN_BLOCK_NUMBER, blockProgressRepo);
            var logProcessor          = new BlockRangeLogsProcessor(web3, eventSubscriptions);
            var batchProcessorService = new LogsProcessor(logProcessor, progressService, MAX_BLOCKS_PER_BATCH);

            // execute
            BlockRange?rangeProcessed;

            try
            {
                var ctx = new System.Threading.CancellationTokenSource();
                rangeProcessed = await batchProcessorService.ProcessOnceAsync(ctx.Token);
            }
            finally
            {
                await ClearDown(configurationContext, storageCloudSetup, searchService, queueFactory, repositoryFactory);
            }

            // save event subscription state
            await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State));

            // assertions
            Assert.NotNull(rangeProcessed);
            Assert.Equal((ulong)10, rangeProcessed.Value.BlockCount);

            var subscriptionState1 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 1); // interested in transfers with contract queries and aggregations
            var subscriptionState2 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 2); // interested in transfers with simple aggregation
            var subscriptionState3 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 3); // interested in any event for a specific address

            Assert.Equal("4009000000002040652615", subscriptionState1.Values["RunningTotalForTransferValue"].ToString());
            Assert.Equal((uint)19, subscriptionState2.Values["CurrentTransferCount"]);

            var txForSpecificAddress = (List <string>)subscriptionState3.Values["AllTransactionHashes"];

            Assert.Equal("0x362bcbc78a5cc6156e8d24d95bee6b8f53d7821083940434d2191feba477ae0e", txForSpecificAddress[0]);
            Assert.Equal("0xe63e9422dedf84d0ce13f9f75ebfd86333ce917b2572925fbdd51b51caf89b77", txForSpecificAddress[1]);

            var blockNumbersForSpecificAddress = (List <HexBigInteger>)subscriptionState3.Values["AllBlockNumbers"];

            Assert.Equal((BigInteger)4063362, blockNumbersForSpecificAddress[0].Value);
            Assert.Equal((BigInteger)4063362, blockNumbersForSpecificAddress[1].Value);
        }