Пример #1
0
        private async Task ClearDown(
            EventProcessingConfigContext repo,
            AzureTablesRepositoryFactory cloudTableSetup,
            IAzureSearchService searchService,
            AzureStorageQueueFactory subscriberQueueFactory,
            AzureTablesSubscriberRepositoryFactory azureTablesSubscriberRepositoryFactory)
        {
            foreach (var index in repo.SubscriberSearchIndexes)
            {
                await searchService.DeleteIndexAsync(index.Name);
            }

            foreach (var queue in repo.SubscriberSearchIndexes)
            {
                await subscriberQueueFactory.DeleteQueueAsync(queue.Name);
            }

            await cloudTableSetup.GetCountersTable().DeleteIfExistsAsync();

            await azureTablesSubscriberRepositoryFactory.DeleteTablesAsync();
        }
        public async Task WriteTransferEventsForMakerDAOToAzureStorage()
        {
            // Load config
            //  - this will contain the secrets and connection strings we don't want to hard code
            var    config = TestConfiguration.LoadConfig();
            string azureStorageConnectionString = config["AzureStorageConnectionString"];

            // Create a proxy for the blockchain
            var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet);

            // Create an Azure Table Storage Factory
            //  - The factory communicates with Azure to create and get different tables
            var tableStorageFactory = new AzureTablesSubscriberRepositoryFactory(azureStorageConnectionString);

            // Create a Handler for a Table
            // - It wraps a table repository
            // - This is where we're going to put the matching event logs
            // - we're supplying a table prefix
            // - the actual table name would be "<prefix>TransactionLogs")
            // - this allows us to have different tables for different types of event logs
            // - the handler implements ILogHandler
            // - ILogHandler is a really simple interface to implement if you wish to customise the storage
            var storageHandlerForLogs = await tableStorageFactory.GetLogRepositoryHandlerAsync(tablePrefix : "makerdaotransfersstorage");

            // Create an event subscription specifically for ERC20 Transfers
            // - Passing in the maker dao address to ensure only logs with a matching address are processed
            // - There is an option to pass an implementation of IEventHandlerHistoryRepository in to the constructor
            // - This would record history for each event handler and is used to prevent duplication
            var eventSubscription = new EventSubscription <TransferEventDto>(
                contractAddressesToMatch: new[] { MAKER_CONTRACT_ADDRESS });

            // Assign the storage handler to the event subscription
            // - Matching events will be passed to the handler
            // - internally the handler passes the events to the repository layer which writes them to Azure
            eventSubscription.AddStorageHandler(storageHandlerForLogs);

            // Azure storage setup
            // - this example reads and writes block progress to an Azure storage table
            // - to avoid collision with other samples we provide a prefix
            var storageCloudSetup = new CloudTableSetup(azureStorageConnectionString, prefix: $"makerdaotransfersstorage");

            // Create a progress repository
            //  - It stores and retrieves the most recent block processed
            var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository();

            // Create a progress service
            // - This uses the progress repo to dictate what blocks to process next
            // - The MIN_BLOCK_NUMBER dictates the starting point if the progress repo is empty or has fallen too far behind
            var progressService = new BlockProgressService(web3, MIN_BLOCK_NUMBER, blockProgressRepo);

            // Create a filter
            //  - This is essentially the query that is sent to the chain when retrieving logs
            //  - It is OPTIONAL - without it, all logs in the block range are requested
            //  - The filter is invoked before any event subscriptions evaluate the logs
            //  - The subscriptions are free to implement their own matching logic
            //  - In this sample we're only interested in MakerDAO logs
            //  - Therefore it makes sense to restrict the number of logs to retrieve from the chain
            var makerAddressFilter = new NewFilterInput()
            {
                Address = new[] { MAKER_CONTRACT_ADDRESS }
            };

            // Create a log processor
            // - This uses the blockchainProxy to get the logs
            // - It sends each log to the event subscriptions to indicate if the logs matches the subscription criteria
            // - It then allocates matching logs to separate batches per event subscription
            var logProcessor = new BlockRangeLogsProcessor(web3, new[] { eventSubscription }, makerAddressFilter);

            // Create a batch log processor service
            // - It uses the progress service to calculates the block range to progress
            // - It then invokes the log processor - passing in the range to process
            // - It updates progress via the progress service
            var batchProcessorService = new LogsProcessor(logProcessor, progressService, MAX_BLOCKS_PER_BATCH);

            // execute
            try
            {
                // Optional cancellation token
                // - Useful for cancelling long running processing operations
                var ctx = new System.Threading.CancellationTokenSource();

                // instruct the service to get and process the next range of blocks
                // when the rangeProcessed is null - it means there was nothing to process
                var rangeProcessed = await batchProcessorService.ProcessOnceAsync(ctx.Token);

                // ensure we have processed the expected number of events
                // the event subscription has state which can record running totals across many processing batches
                Assert.Equal(11, eventSubscription.State.GetInt("EventsHandled"));

                // get the row count from azure storage
                // the querying on storage is limited
                // the TransactionHash is the partitionkey and the rowkey is the LogIndex
                // this allows us to query by tx hash

                var logRepositoryHandler = storageHandlerForLogs as TransactionLogRepositoryHandler;
                var repository           = logRepositoryHandler.TransactionLogRepository as TransactionLogRepository;

                var expectedTransactionHashes = new[]
                {
                    "0x8d58abc578f5e321f2e6b7c0637ccc60fbf62b39b120691cbf19ff201f5069b0",
                    "0x0bee561ac6bafb59bcc4c48fc4c1225aaedbab3e8089acea420140aafa47f3e5",
                    "0x6fc82b076fa7088581a80869cb9c7a08d7f8e897670a9f67e39139b39246da7e",
                    "0xdc2ee28db35ed5dbbc9e18a7d6bdbacb6e6633a9fce1ecda99ea7e1cf4bc8c72",
                    "0xcd2fea48c84468f70c9a44c4ffd7b26064a2add8b72937edf593634d2501c1f6",
                    "0x3acf887420887148222aab1d25d4d4893794e505ef276cc4cb6a48fffc6cb381",
                    "0x96129f905589b2a95c26276aa7e8708a12381ddec50485d6684c4abf9a5a1d00"
                };

                List <TransactionLog> logsFromRepo = new List <TransactionLog>();
                foreach (var txHash in expectedTransactionHashes)
                {
                    logsFromRepo.AddRange(await repository.GetManyAsync(txHash));
                }

                Assert.Equal(11, logsFromRepo.Count);
            }
            finally
            {
                // delete any data from Azure
                await storageCloudSetup.GetCountersTable().DeleteIfExistsAsync();

                await tableStorageFactory.DeleteTablesAsync();
            }
        }
Пример #3
0
        public async Task WebJobExample()
        {
            var    config = TestConfiguration.LoadConfig();
            string azureStorageConnectionString = config["AzureStorageConnectionString"];
            string azureSearchKey = config["AzureSearchApiKey"];

            var configurationContext = EventProcessingConfigMock.Create(PARTITION, out IdGenerator idGenerator);
            IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator);

            var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby);

            // search components
            var searchService = new AzureSearchService(serviceName: AZURE_SEARCH_SERVICE_NAME, searchApiKey: azureSearchKey);

            var subscriberSearchIndexFactory = new SubscriberSearchIndexFactory(async indexName =>
            {
                if (await searchService.IndexExistsAsync(indexName) == false)
                {
                    //TODO: REPLACE THIS WITH Nethereum.BlockchainStore.Search.Azure.EventToGenericSearchDocMapper

                    await searchService.CreateIndexAsync(EventToGenericSearchDocMapper.CreateAzureIndexDefinition(indexName));
                }

                return(searchService.CreateIndexer <DecodedEvent, GenericSearchDocument>(
                           indexName, decodedEvent => EventToGenericSearchDocMapper.Map(decodedEvent, decodedEvent.State)));
            });

            // queue components
            //AzureStorageQueueFactory
            var azureQueueFactory = new AzureStorageQueueFactory(azureStorageConnectionString);

            var subscriberQueueFactory = new SubscriberQueueFactory(
                queueName => azureQueueFactory.GetOrCreateQueueAsync(queueName));

            // subscriber repository
            var repositoryFactory = new AzureTablesSubscriberRepositoryFactory(azureStorageConnectionString);

            // load subscribers and event subscriptions
            var eventSubscriptionFactory = new EventSubscriptionFactory(
                web3, configurationRepository, subscriberQueueFactory, subscriberSearchIndexFactory, repositoryFactory);

            var eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION);

            // progress repo (dictates which block ranges to process next)
            // maintain separate progress per partition via a prefix
            var storageCloudSetup = new AzureTablesRepositoryFactory(azureStorageConnectionString, prefix: $"Partition{PARTITION}");
            var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository();

            var logProcessor = web3.Processing.Logs.CreateProcessor(
                logProcessors: eventSubscriptions, blockProgressRepository: blockProgressRepo);

            // execute
            try
            {
                var ctx = new System.Threading.CancellationTokenSource();
                await logProcessor.ExecuteAsync(BLOCK_TO, ctx.Token, BLOCK_FROM);
            }
            finally
            {
                await ClearDown(configurationContext, storageCloudSetup, searchService, azureQueueFactory, repositoryFactory);
            }

            // save event subscription state
            await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State));

            // assertions

            var subscriptionState1 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 1); // interested in transfers with contract queries and aggregations
            var subscriptionState2 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 2); // interested in transfers with simple aggregation
            var subscriptionState3 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 3); // interested in any event for a specific address

            Assert.Equal("4009000000002040652615", subscriptionState1.Values["RunningTotalForTransferValue"].ToString());
            Assert.Equal((uint)19, subscriptionState2.Values["CurrentTransferCount"]);

            var txForSpecificAddress = (List <string>)subscriptionState3.Values["AllTransactionHashes"];

            Assert.Equal("0x362bcbc78a5cc6156e8d24d95bee6b8f53d7821083940434d2191feba477ae0e", txForSpecificAddress[0]);
            Assert.Equal("0xe63e9422dedf84d0ce13f9f75ebfd86333ce917b2572925fbdd51b51caf89b77", txForSpecificAddress[1]);

            var blockNumbersForSpecificAddress = (List <HexBigInteger>)subscriptionState3.Values["AllBlockNumbers"];

            Assert.Equal(4063362, blockNumbersForSpecificAddress[0].Value);
            Assert.Equal(4063362, blockNumbersForSpecificAddress[1].Value);
        }
        public async Task WebJobExample()
        {
            var    config = TestConfiguration.LoadConfig();
            string azureStorageConnectionString = config["AzureStorageConnectionString"];
            string azureSearchKey = config["AzureSearchApiKey"];

            var configurationContext = EventProcessingConfigMock.Create(PARTITION, out IdGenerator idGenerator);
            IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator);

            var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby);

            // search components
            var searchService      = new AzureSearchService(serviceName: AZURE_SEARCH_SERVICE_NAME, searchApiKey: azureSearchKey);
            var searchIndexFactory = new AzureSubscriberSearchIndexFactory(searchService);

            // queue components
            var queueFactory = new AzureSubscriberQueueFactory(azureStorageConnectionString);

            // subscriber repository
            var repositoryFactory = new AzureTablesSubscriberRepositoryFactory(azureStorageConnectionString);

            // load subscribers and event subscriptions
            var eventSubscriptionFactory = new EventSubscriptionFactory(
                web3, configurationRepository, queueFactory, searchIndexFactory, repositoryFactory);

            List <IEventSubscription> eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION);

            // progress repo (dictates which block ranges to process next)
            // maintain separate progress per partition via a prefix
            var storageCloudSetup = new CloudTableSetup(azureStorageConnectionString, prefix: $"Partition{PARTITION}");
            var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository();

            // load service
            var progressService       = new BlockProgressService(web3, MIN_BLOCK_NUMBER, blockProgressRepo);
            var logProcessor          = new BlockRangeLogsProcessor(web3, eventSubscriptions);
            var batchProcessorService = new LogsProcessor(logProcessor, progressService, MAX_BLOCKS_PER_BATCH);

            // execute
            BlockRange?rangeProcessed;

            try
            {
                var ctx = new System.Threading.CancellationTokenSource();
                rangeProcessed = await batchProcessorService.ProcessOnceAsync(ctx.Token);
            }
            finally
            {
                await ClearDown(configurationContext, storageCloudSetup, searchService, queueFactory, repositoryFactory);
            }

            // save event subscription state
            await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State));

            // assertions
            Assert.NotNull(rangeProcessed);
            Assert.Equal((ulong)10, rangeProcessed.Value.BlockCount);

            var subscriptionState1 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 1); // interested in transfers with contract queries and aggregations
            var subscriptionState2 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 2); // interested in transfers with simple aggregation
            var subscriptionState3 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 3); // interested in any event for a specific address

            Assert.Equal("4009000000002040652615", subscriptionState1.Values["RunningTotalForTransferValue"].ToString());
            Assert.Equal((uint)19, subscriptionState2.Values["CurrentTransferCount"]);

            var txForSpecificAddress = (List <string>)subscriptionState3.Values["AllTransactionHashes"];

            Assert.Equal("0x362bcbc78a5cc6156e8d24d95bee6b8f53d7821083940434d2191feba477ae0e", txForSpecificAddress[0]);
            Assert.Equal("0xe63e9422dedf84d0ce13f9f75ebfd86333ce917b2572925fbdd51b51caf89b77", txForSpecificAddress[1]);

            var blockNumbersForSpecificAddress = (List <HexBigInteger>)subscriptionState3.Values["AllBlockNumbers"];

            Assert.Equal((BigInteger)4063362, blockNumbersForSpecificAddress[0].Value);
            Assert.Equal((BigInteger)4063362, blockNumbersForSpecificAddress[1].Value);
        }