public async Task AnyContractAnyLog() { const string QUEUE_NAME = "any-contract-any-log"; var azureQueueFactory = new AzureStorageQueueFactory(_azureConnectionString); try { var queue = await azureQueueFactory.GetOrCreateQueueAsync(QUEUE_NAME); var logProcessor = _web3.Processing.Logs.CreateProcessor(filterLog => queue.AddMessageAsync(filterLog)); //if we need to stop the processor mid execution - call cancel on the token var cancellationTokenSource = new CancellationTokenSource(); //crawl the required block range await logProcessor.ExecuteAsync( toBlockNumber : 3146690, cancellationToken : cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed : 3146684); Assert.Equal(65, await queue.GetApproxMessageCountAsync()); } finally { await azureQueueFactory.DeleteQueueAsync(QUEUE_NAME); } }
public async Task MappingAnEventLogToACustomQueueMessage() { const string QUEUE_NAME = "mapping-from-event-log"; var azureQueueFactory = new AzureStorageQueueFactory(_azureConnectionString); try { var queue = await azureQueueFactory.GetOrCreateQueueAsync <EventLog <TransferEventDTO>, MessageToQueue>( QUEUE_NAME, transferEvent => new MessageToQueue(transferEvent)); var logProcessor = _web3.Processing.Logs.CreateProcessor <TransferEventDTO>( transferEvent => queue.AddMessageAsync(transferEvent)); //if we need to stop the processor mid execution - call cancel on the token var cancellationTokenSource = new CancellationTokenSource(); //crawl the required block range await logProcessor.ExecuteAsync( toBlockNumber : 3146690, cancellationToken : cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed : 3146684); Assert.Equal(13, await queue.GetApproxMessageCountAsync()); } finally { await azureQueueFactory.DeleteQueueAsync(QUEUE_NAME); } }
public async Task EventSpecificCriteria() { const string QUEUE_NAME = "with-event-specific-criteria"; var azureQueueFactory = new AzureStorageQueueFactory(_azureConnectionString); try { var queue = await azureQueueFactory.GetOrCreateQueueAsync(QUEUE_NAME); var minValue = BigInteger.Parse("5000000000000000000"); var logProcessor = _web3.Processing.Logs.CreateProcessor <TransferEventDTO>( action: transferEvent => queue.AddMessageAsync(transferEvent), criteria: transferEvent => transferEvent.Event.Value >= minValue); //if we need to stop the processor mid execution - call cancel on the token var cancellationTokenSource = new CancellationTokenSource(); //crawl the required block range await logProcessor.ExecuteAsync( toBlockNumber : 3146690, cancellationToken : cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed : 3146684); await Task.Delay(2000); //give time for queue to update Assert.Equal(6, await queue.GetApproxMessageCountAsync()); } finally { await azureQueueFactory.DeleteQueueAsync(QUEUE_NAME); } }
public async Task ManyContractsOneEvent() { const string QUEUE_NAME = "many-contracts-one-event"; var azureQueueFactory = new AzureStorageQueueFactory(_azureConnectionString); try { var queue = await azureQueueFactory.GetOrCreateQueueAsync(QUEUE_NAME); var contractAddresses = new[] { "0x109424946d5aa4425b2dc1934031d634cdad3f90", "0x16c45b25c4817bdedfce770f795790795c9505a6" }; var logProcessor = _web3.Processing.Logs.CreateProcessorForContracts <TransferEventDTO>( contractAddresses, transferEvent => queue.AddMessageAsync(transferEvent)); //if we need to stop the processor mid execution - call cancel on the token var cancellationTokenSource = new CancellationTokenSource(); //crawl the required block range await logProcessor.ExecuteAsync( toBlockNumber : 3146690, cancellationToken : cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed : 3146684); Assert.Equal(5, await queue.GetApproxMessageCountAsync()); } finally { await azureQueueFactory.DeleteQueueAsync(QUEUE_NAME); } }
public async Task OneContractAnyLog() { const string QUEUE_NAME = "one-contract-any-log"; var azureQueueFactory = new AzureStorageQueueFactory(_azureConnectionString); try { var queue = await azureQueueFactory.GetOrCreateQueueAsync(QUEUE_NAME); var contractFilter = new NewFilterInput { Address = new[] { "0x109424946d5aa4425b2dc1934031d634cdad3f90" } }; var logProcessor = _web3.Processing.Logs.CreateProcessor( action: filterLog => queue.AddMessageAsync(filterLog), filter: contractFilter); //if we need to stop the processor mid execution - call cancel on the token var cancellationTokenSource = new CancellationTokenSource(); //crawl the required block range await logProcessor.ExecuteAsync( toBlockNumber : 3146690, cancellationToken : cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed : 3146684); Assert.Equal(4, await queue.GetApproxMessageCountAsync()); } finally { await azureQueueFactory.DeleteQueueAsync(QUEUE_NAME); } }
public void TestInitialize() { // Arrange var azureStorageQueueConfig = new AzureStorageQueueConfig { ConnectionString = AzureStorageData.ConnectionString, QueueName = "osw-lib-dataaccess-azurestorage" }; this.azureStorageQueue = AzureStorageQueueFactory.Create( this.LoggerFactoryEnriched, azureStorageQueueConfig); }
public async Task ManyContractsMultipleEvents() { const string ERC20_QUEUE_NAME = "many-contract-multi-event-erc20"; const string APPROVAL_QUEUE_NAME = "many-contract-multi-event-approval"; var azureQueueFactory = new AzureStorageQueueFactory(_azureConnectionString); try { var erc20Queue = await azureQueueFactory.GetOrCreateQueueAsync(ERC20_QUEUE_NAME); var approvalQueue = await azureQueueFactory.GetOrCreateQueueAsync(APPROVAL_QUEUE_NAME); var erc20TransferProcessor = new EventLogProcessorHandler <TransferEventDTO>(transfer => erc20Queue.AddMessageAsync(transfer)); var approvalProcessor = new EventLogProcessorHandler <ApprovalEventDTO>(approval => approvalQueue.AddMessageAsync(approval)); var logProcessors = new ProcessorHandler <FilterLog>[] { erc20TransferProcessor, approvalProcessor }; var contractAddresses = new[] { "0x9EDCb9A9c4d34b5d6A082c86cb4f117A1394F831", "0xafbfefa496ae205cf4e002dee11517e6d6da3ef6" }; var contractFilter = new NewFilterInput { Address = contractAddresses }; var logProcessor = _web3.Processing.Logs.CreateProcessor(logProcessors, filter: contractFilter); //if we need to stop the processor mid execution - call cancel on the token var cancellationTokenSource = new CancellationTokenSource(); //crawl the required block range await logProcessor.ExecuteAsync( toBlockNumber : 3621716, cancellationToken : cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed : 3621715); Assert.Equal(2, await erc20Queue.GetApproxMessageCountAsync()); Assert.Equal(1, await approvalQueue.GetApproxMessageCountAsync()); } finally { await azureQueueFactory.DeleteQueueAsync(ERC20_QUEUE_NAME); await azureQueueFactory.DeleteQueueAsync(APPROVAL_QUEUE_NAME); } }
public void TestInitialize() { // Arrange var azureStorageQueueConfig = new AzureStorageQueueConfig { ConnectionString = AzureStorageData.ConnectionString, QueueName = "osw-lib-dataaccess-azurestorage" }; this.azureStorageQueue = AzureStorageQueueFactory.Create( this.LoggerFactoryEnriched, azureStorageQueueConfig); this.azureStorageQueue.CreateIfNotExists(); this.testQueueMessage = new TestQueueMessage { Author = OrisicSoftworks }; }
private async Task ClearDown( EventProcessingConfigContext repo, AzureTablesRepositoryFactory cloudTableSetup, IAzureSearchService searchService, AzureStorageQueueFactory subscriberQueueFactory, AzureTablesSubscriberRepositoryFactory azureTablesSubscriberRepositoryFactory) { foreach (var index in repo.SubscriberSearchIndexes) { await searchService.DeleteIndexAsync(index.Name); } foreach (var queue in repo.SubscriberSearchIndexes) { await subscriberQueueFactory.DeleteQueueAsync(queue.Name); } await cloudTableSetup.GetCountersTable().DeleteIfExistsAsync(); await azureTablesSubscriberRepositoryFactory.DeleteTablesAsync(); }
public async Task AnyContractMultipleEvents() { const string ERC20_QUEUE_NAME = "any-contract-one-event-erc20"; const string ERC721_QUEUE_NAME = "any-contract-one-event-erc721"; var azureQueueFactory = new AzureStorageQueueFactory(_azureConnectionString); try { var erc20Queue = await azureQueueFactory.GetOrCreateQueueAsync(ERC20_QUEUE_NAME); var erc721Queue = await azureQueueFactory.GetOrCreateQueueAsync(ERC721_QUEUE_NAME); var erc20TransferProcessor = new EventLogProcessorHandler <TransferEventDTO>(transfer => erc20Queue.AddMessageAsync(transfer)); var erc721TransferProcessor = new EventLogProcessorHandler <Erc721TransferEvent>(transfer => erc721Queue.AddMessageAsync(transfer)); var logProcessors = new ProcessorHandler <FilterLog>[] { erc20TransferProcessor, erc721TransferProcessor }; var logProcessor = _web3.Processing.Logs.CreateProcessor(logProcessors); //if we need to stop the processor mid execution - call cancel on the token var cancellationTokenSource = new CancellationTokenSource(); //crawl the required block range await logProcessor.ExecuteAsync( toBlockNumber : 3146690, cancellationToken : cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed : 3146684); Assert.Equal(13, await erc20Queue.GetApproxMessageCountAsync()); Assert.Equal(3, await erc721Queue.GetApproxMessageCountAsync()); } finally { await azureQueueFactory.DeleteQueueAsync(ERC20_QUEUE_NAME); await azureQueueFactory.DeleteQueueAsync(ERC721_QUEUE_NAME); } }
public async Task WebJobExample() { var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; string azureSearchKey = config["AzureSearchApiKey"]; var configurationContext = EventProcessingConfigMock.Create(PARTITION, out IdGenerator idGenerator); IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator); var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby); // search components var searchService = new AzureSearchService(serviceName: AZURE_SEARCH_SERVICE_NAME, searchApiKey: azureSearchKey); var subscriberSearchIndexFactory = new SubscriberSearchIndexFactory(async indexName => { if (await searchService.IndexExistsAsync(indexName) == false) { //TODO: REPLACE THIS WITH Nethereum.BlockchainStore.Search.Azure.EventToGenericSearchDocMapper await searchService.CreateIndexAsync(EventToGenericSearchDocMapper.CreateAzureIndexDefinition(indexName)); } return(searchService.CreateIndexer <DecodedEvent, GenericSearchDocument>( indexName, decodedEvent => EventToGenericSearchDocMapper.Map(decodedEvent, decodedEvent.State))); }); // queue components //AzureStorageQueueFactory var azureQueueFactory = new AzureStorageQueueFactory(azureStorageConnectionString); var subscriberQueueFactory = new SubscriberQueueFactory( queueName => azureQueueFactory.GetOrCreateQueueAsync(queueName)); // subscriber repository var repositoryFactory = new AzureTablesSubscriberRepositoryFactory(azureStorageConnectionString); // load subscribers and event subscriptions var eventSubscriptionFactory = new EventSubscriptionFactory( web3, configurationRepository, subscriberQueueFactory, subscriberSearchIndexFactory, repositoryFactory); var eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION); // progress repo (dictates which block ranges to process next) // maintain separate progress per partition via a prefix var storageCloudSetup = new AzureTablesRepositoryFactory(azureStorageConnectionString, prefix: $"Partition{PARTITION}"); var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository(); var logProcessor = web3.Processing.Logs.CreateProcessor( logProcessors: eventSubscriptions, blockProgressRepository: blockProgressRepo); // execute try { var ctx = new System.Threading.CancellationTokenSource(); await logProcessor.ExecuteAsync(BLOCK_TO, ctx.Token, BLOCK_FROM); } finally { await ClearDown(configurationContext, storageCloudSetup, searchService, azureQueueFactory, repositoryFactory); } // save event subscription state await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State)); // assertions var subscriptionState1 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 1); // interested in transfers with contract queries and aggregations var subscriptionState2 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 2); // interested in transfers with simple aggregation var subscriptionState3 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 3); // interested in any event for a specific address Assert.Equal("4009000000002040652615", subscriptionState1.Values["RunningTotalForTransferValue"].ToString()); Assert.Equal((uint)19, subscriptionState2.Values["CurrentTransferCount"]); var txForSpecificAddress = (List <string>)subscriptionState3.Values["AllTransactionHashes"]; Assert.Equal("0x362bcbc78a5cc6156e8d24d95bee6b8f53d7821083940434d2191feba477ae0e", txForSpecificAddress[0]); Assert.Equal("0xe63e9422dedf84d0ce13f9f75ebfd86333ce917b2572925fbdd51b51caf89b77", txForSpecificAddress[1]); var blockNumbersForSpecificAddress = (List <HexBigInteger>)subscriptionState3.Values["AllBlockNumbers"]; Assert.Equal(4063362, blockNumbersForSpecificAddress[0].Value); Assert.Equal(4063362, blockNumbersForSpecificAddress[1].Value); }