public AzureTablesConfigurationRepoFixture() { var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; CloudTableSetup = new EventProcessingCloudTableSetup(azureStorageConnectionString, "test"); ConfigRepo = new AzureEventProcessingConfigurationRepository(CloudTableSetup); }
public EventSubscriptionFactory( IEventProcessingConfigurationRepository configurationRepository, IEventMatcherFactory eventMatcherFactory, IEventHandlerFactory eventHandlerFactory) { ConfigurationRepository = configurationRepository; EventMatcherFactory = eventMatcherFactory; EventHandlerFactory = eventHandlerFactory; }
public EventSubscriptionFactory( IWeb3 web3, IEventProcessingConfigurationRepository configurationRepository, ISubscriberQueueFactory subscriberQueueFactory = null, ISubscriberSearchIndexFactory subscriberSearchIndexFactory = null, ISubscriberStorageFactory subscriberRepositoryFactory = null) : this( configurationRepository, new EventMatcherFactory( configurationRepository.ParameterConditions, configurationRepository.EventSubscriptionAddresses, configurationRepository.SubscriberContracts), new EventHandlerFactory( web3, configurationRepository, subscriberQueueFactory, subscriberSearchIndexFactory, subscriberRepositoryFactory)) { }
public EventHandlerFactory( IWeb3 web3, IEventProcessingConfigurationRepository configRepo, ISubscriberQueueFactory subscriberQueueFactory = null, ISubscriberSearchIndexFactory subscriberSearchIndexFactory = null, ISubscriberStorageFactory subscriberRepositoryFactory = null) : this( web3.Eth, configRepo.EventSubscriptionStates, configRepo.EventContractQueries, configRepo.EventAggregators, web3.Eth.Transactions.GetTransactionByHash, configRepo.SubscriberQueues, subscriberQueueFactory, configRepo.SubscriberSearchIndexes, subscriberSearchIndexFactory, configRepo.EventRules, configRepo.SubscriberStorage, subscriberRepositoryFactory) { }
public async Task WriteAnyMakerEventToQueue() { var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; var configurationContext = MakerDAOEventProcessingConfig.Create(PARTITION, out IdGenerator idGenerator); IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator); var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet); // queue components var queueFactory = new AzureSubscriberQueueFactory(azureStorageConnectionString); // load subscribers and event subscriptions var eventSubscriptionFactory = new EventSubscriptionFactory( web3, configurationRepository, queueFactory); List <IEventSubscription> eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION); // progress repo (dictates which block ranges to process next) // maintain separate progress per partition via a prefix var storageCloudSetup = new CloudTableSetup(azureStorageConnectionString, prefix: $"Partition{PARTITION}"); var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository(); //this ensures we only query the chain for events relating to this contract var makerAddressFilter = new NewFilterInput() { Address = new[] { MakerDAOEventProcessingConfig.MAKER_CONTRACT_ADDRESS } }; // load service var progressService = new BlockProgressService(web3, MIN_BLOCK_NUMBER, blockProgressRepo); var logProcessor = new BlockRangeLogsProcessor(web3, eventSubscriptions, makerAddressFilter); var batchProcessorService = new LogsProcessor(logProcessor, progressService, MAX_BLOCKS_PER_BATCH); // execute var blockRangesProcessed = new List <BlockRange?>(); try { for (var i = 0; i < 2; i++) // 2 batch iterations { var ctx = new System.Threading.CancellationTokenSource(); var rangeProcessed = await batchProcessorService.ProcessOnceAsync(ctx.Token); blockRangesProcessed.Add(rangeProcessed); // save event subscription state after each batch await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State)); } } finally { await ClearDown(configurationContext, storageCloudSetup, queueFactory); } var subscriptionState = await configurationRepository.EventSubscriptionStates.GetAsync(eventSubscriptions[0].Id); Assert.Equal(2, (int)subscriptionState.Values["HandlerInvocations"]); Assert.Equal(28, (int)subscriptionState.Values["EventsHandled"]); }
public async Task WebJobExample() { var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; string azureSearchKey = config["AzureSearchApiKey"]; var configurationContext = EventProcessingConfigMock.Create(PARTITION, out IdGenerator idGenerator); IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator); var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby); // search components var searchService = new AzureSearchService(serviceName: AZURE_SEARCH_SERVICE_NAME, searchApiKey: azureSearchKey); var subscriberSearchIndexFactory = new SubscriberSearchIndexFactory(async indexName => { if (await searchService.IndexExistsAsync(indexName) == false) { //TODO: REPLACE THIS WITH Nethereum.BlockchainStore.Search.Azure.EventToGenericSearchDocMapper await searchService.CreateIndexAsync(EventToGenericSearchDocMapper.CreateAzureIndexDefinition(indexName)); } return(searchService.CreateIndexer <DecodedEvent, GenericSearchDocument>( indexName, decodedEvent => EventToGenericSearchDocMapper.Map(decodedEvent, decodedEvent.State))); }); // queue components //AzureStorageQueueFactory var azureQueueFactory = new AzureStorageQueueFactory(azureStorageConnectionString); var subscriberQueueFactory = new SubscriberQueueFactory( queueName => azureQueueFactory.GetOrCreateQueueAsync(queueName)); // subscriber repository var repositoryFactory = new AzureTablesSubscriberRepositoryFactory(azureStorageConnectionString); // load subscribers and event subscriptions var eventSubscriptionFactory = new EventSubscriptionFactory( web3, configurationRepository, subscriberQueueFactory, subscriberSearchIndexFactory, repositoryFactory); var eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION); // progress repo (dictates which block ranges to process next) // maintain separate progress per partition via a prefix var storageCloudSetup = new AzureTablesRepositoryFactory(azureStorageConnectionString, prefix: $"Partition{PARTITION}"); var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository(); var logProcessor = web3.Processing.Logs.CreateProcessor( logProcessors: eventSubscriptions, blockProgressRepository: blockProgressRepo); // execute try { var ctx = new System.Threading.CancellationTokenSource(); await logProcessor.ExecuteAsync(BLOCK_TO, ctx.Token, BLOCK_FROM); } finally { await ClearDown(configurationContext, storageCloudSetup, searchService, azureQueueFactory, repositoryFactory); } // save event subscription state await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State)); // assertions var subscriptionState1 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 1); // interested in transfers with contract queries and aggregations var subscriptionState2 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 2); // interested in transfers with simple aggregation var subscriptionState3 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 3); // interested in any event for a specific address Assert.Equal("4009000000002040652615", subscriptionState1.Values["RunningTotalForTransferValue"].ToString()); Assert.Equal((uint)19, subscriptionState2.Values["CurrentTransferCount"]); var txForSpecificAddress = (List <string>)subscriptionState3.Values["AllTransactionHashes"]; Assert.Equal("0x362bcbc78a5cc6156e8d24d95bee6b8f53d7821083940434d2191feba477ae0e", txForSpecificAddress[0]); Assert.Equal("0xe63e9422dedf84d0ce13f9f75ebfd86333ce917b2572925fbdd51b51caf89b77", txForSpecificAddress[1]); var blockNumbersForSpecificAddress = (List <HexBigInteger>)subscriptionState3.Values["AllBlockNumbers"]; Assert.Equal(4063362, blockNumbersForSpecificAddress[0].Value); Assert.Equal(4063362, blockNumbersForSpecificAddress[1].Value); }
public async Task WebJobExample() { var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; string azureSearchKey = config["AzureSearchApiKey"]; var configurationContext = EventProcessingConfigMock.Create(PARTITION, out IdGenerator idGenerator); IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator); var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby); // search components var searchService = new AzureSearchService(serviceName: AZURE_SEARCH_SERVICE_NAME, searchApiKey: azureSearchKey); var searchIndexFactory = new AzureSubscriberSearchIndexFactory(searchService); // queue components var queueFactory = new AzureSubscriberQueueFactory(azureStorageConnectionString); // subscriber repository var repositoryFactory = new AzureTablesSubscriberRepositoryFactory(azureStorageConnectionString); // load subscribers and event subscriptions var eventSubscriptionFactory = new EventSubscriptionFactory( web3, configurationRepository, queueFactory, searchIndexFactory, repositoryFactory); List <IEventSubscription> eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION); // progress repo (dictates which block ranges to process next) // maintain separate progress per partition via a prefix var storageCloudSetup = new CloudTableSetup(azureStorageConnectionString, prefix: $"Partition{PARTITION}"); var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository(); // load service var progressService = new BlockProgressService(web3, MIN_BLOCK_NUMBER, blockProgressRepo); var logProcessor = new BlockRangeLogsProcessor(web3, eventSubscriptions); var batchProcessorService = new LogsProcessor(logProcessor, progressService, MAX_BLOCKS_PER_BATCH); // execute BlockRange?rangeProcessed; try { var ctx = new System.Threading.CancellationTokenSource(); rangeProcessed = await batchProcessorService.ProcessOnceAsync(ctx.Token); } finally { await ClearDown(configurationContext, storageCloudSetup, searchService, queueFactory, repositoryFactory); } // save event subscription state await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State)); // assertions Assert.NotNull(rangeProcessed); Assert.Equal((ulong)10, rangeProcessed.Value.BlockCount); var subscriptionState1 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 1); // interested in transfers with contract queries and aggregations var subscriptionState2 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 2); // interested in transfers with simple aggregation var subscriptionState3 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 3); // interested in any event for a specific address Assert.Equal("4009000000002040652615", subscriptionState1.Values["RunningTotalForTransferValue"].ToString()); Assert.Equal((uint)19, subscriptionState2.Values["CurrentTransferCount"]); var txForSpecificAddress = (List <string>)subscriptionState3.Values["AllTransactionHashes"]; Assert.Equal("0x362bcbc78a5cc6156e8d24d95bee6b8f53d7821083940434d2191feba477ae0e", txForSpecificAddress[0]); Assert.Equal("0xe63e9422dedf84d0ce13f9f75ebfd86333ce917b2572925fbdd51b51caf89b77", txForSpecificAddress[1]); var blockNumbersForSpecificAddress = (List <HexBigInteger>)subscriptionState3.Values["AllBlockNumbers"]; Assert.Equal((BigInteger)4063362, blockNumbersForSpecificAddress[0].Value); Assert.Equal((BigInteger)4063362, blockNumbersForSpecificAddress[1].Value); }