public static async Task<ILogsProcessorBuilder> AddToQueueAsync( this ILogsProcessorBuilder eventLogProcessor, string azureConnectionString, string queueName, Predicate<FilterLog> predicate = null, Func<FilterLog, object> mapper = null) { var factory = new AzureSubscriberQueueFactory(azureConnectionString); return await eventLogProcessor.AddToQueueAsync(factory, queueName, predicate, mapper); }
/* * A sample message from the queue * the key is block number, transaction hash and log index * the message contains the raw log (FilterLog) and the decoded parameter values * it also contains extra state data that the processing chain can add or amend * the message can be retrieved from the queue and can be decoded into an event DTO (e.g. TransferEventDto) * { * "Key": "7540010_0x8d58abc578f5e321f2e6b7c0637ccc60fbf62b39b120691cbf19ff201f5069b0_132", * "State": { * "EventAbiName": "Transfer", * "EventSignature": "ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", * "TransactionHash": "0x8d58abc578f5e321f2e6b7c0637ccc60fbf62b39b120691cbf19ff201f5069b0", * "LogIndex": 132, * "HandlerInvocations": 1, * "UtcNowMs": 1556800544598, * "SubscriberId": 0, * "EventSubscriptionId": 0 * }, * "Transaction": null, * "ParameterValues": [{ * "Order": 1, * "Name": "_from", * "AbiType": "address", * "Value": "0x296c61eaf5bea208bbabc65ae01c3bc5270fe386", * "Indexed": true * }, { * "Order": 2, * "Name": "_to", * "AbiType": "address", * "Value": "0x2a8f1a6af55b705b7daee0776d6f97302de2a839", * "Indexed": true * }, { * "Order": 3, * "Name": "_value", * "AbiType": "uint256", * "Value": 119928660890733235, * "Indexed": false * } * ], * "Log": { * "removed": false, * "type": null, * "logIndex": "0x84", * "transactionHash": "0x8d58abc578f5e321f2e6b7c0637ccc60fbf62b39b120691cbf19ff201f5069b0", * "transactionIndex": "0x95", * "blockHash": "0x0ef49c4e3a11872084715808df7a07c4fa6f03dac0596b4e6c10b21b88f7d227", * "blockNumber": "0x730d2a", * "address": "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2", * "data": "0x00000000000000000000000000000000000000000000000001aa127b4ec7cab3", * "topics": ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", "0x000000000000000000000000296c61eaf5bea208bbabc65ae01c3bc5270fe386", "0x0000000000000000000000002a8f1a6af55b705b7daee0776d6f97302de2a839"] * } * } * */ private async Task ClearDown( IQueue queue, CloudTableSetup cloudTableSetup, AzureSubscriberQueueFactory subscriberQueueFactory) { var qRef = subscriberQueueFactory.CloudQueueClient.GetQueueReference(queue.Name); await qRef.DeleteIfExistsAsync(); await cloudTableSetup.GetCountersTable().DeleteIfExistsAsync(); }
public static async Task<ILogsProcessorBuilder> AddToQueueAsync<TEventDto>( this ILogsProcessorBuilder eventLogProcessor, string azureConnectionString, string queueName, Predicate<EventLog<TEventDto>> predicate = null, Func<EventLog<TEventDto>, object> mapper = null) where TEventDto : class, new() { var factory = new AzureSubscriberQueueFactory(azureConnectionString); return await eventLogProcessor.AddToQueueAsync(factory, queueName, predicate, mapper); }
public static async Task<ILogsProcessorBuilder> AddToQueueAsync( this ILogsProcessorBuilder eventLogProcessor, AzureSubscriberQueueFactory queueFactory, string queueName, Predicate<FilterLog> predicate = null, Func<FilterLog, object> mapper = null) { var queue = await queueFactory.GetOrCreateQueueAsync(queueName); eventLogProcessor.AddToQueue(queue, predicate, mapper); return eventLogProcessor; }
public static async Task<ILogsProcessorBuilder> AddToQueueAsync<TEventDto>( this ILogsProcessorBuilder eventLogProcessor, AzureSubscriberQueueFactory queueFactory, string queueName, Predicate<EventLog<TEventDto>> predicate = null, Func<EventLog<TEventDto>, object> mapper = null) where TEventDto : class, new() { var queue = await queueFactory.GetOrCreateQueueAsync(queueName); eventLogProcessor.AddToQueue(queue, predicate, mapper); return eventLogProcessor; }
private async Task ClearDown( EventProcessingConfigContext repo, CloudTableSetup cloudTableSetup, AzureSubscriberQueueFactory subscriberQueueFactory) { foreach (var queue in repo.SubscriberSearchIndexes) { var qRef = subscriberQueueFactory.CloudQueueClient.GetQueueReference(queue.Name); await qRef.DeleteIfExistsAsync(); } await cloudTableSetup.GetCountersTable().DeleteIfExistsAsync(); }
private async Task ClearDown( EventProcessingConfigContext repo, CloudTableSetup cloudTableSetup, IAzureSearchService searchService, AzureSubscriberQueueFactory subscriberQueueFactory, AzureTablesSubscriberRepositoryFactory azureTablesSubscriberRepositoryFactory) { foreach (var index in repo.SubscriberSearchIndexes) { await searchService.DeleteIndexAsync(index.Name); } foreach (var queue in repo.SubscriberSearchIndexes) { var qRef = subscriberQueueFactory.CloudQueueClient.GetQueueReference(queue.Name); await qRef.DeleteIfExistsAsync(); } await cloudTableSetup.GetCountersTable().DeleteIfExistsAsync(); await azureTablesSubscriberRepositoryFactory.DeleteTablesAsync(); }
public async Task WritingEventsToAnAzureQueue() { var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet); // Requires: Nethereum.BlockchainProcessing.Queue.Azure // Load config // - this will contain the secrets and connection strings we don't want to hard code var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; //cancellation token to enable the listener to be stopped //passing in a time limit as a safety valve for the unit test var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1)); //initialise the processor using (var processor = web3.Eth.LogsProcessor <TransferEventDto>() .SetBlocksPerBatch(1) //optional: restrict batches to one block at a time .SetMinimumBlockNumber(7540103) //optional: default is to start at current block on chain .OnBatchProcessed((args) => cancellationTokenSource.Cancel()) .AddToQueueAsync(azureStorageConnectionString, "sep-transfers") .Result .Build()) { //run the processor for a while var rangesProcessed = await processor.ProcessContinuallyAsync(cancellationTokenSource.Token); } await Task.Delay(5000); //give azure time to update //clean up var queueFactory = new AzureSubscriberQueueFactory(azureStorageConnectionString); var queue = await queueFactory.GetOrCreateQueueAsync("sep-transfers"); Assert.Equal(13, await queue.GetApproxMessageCountAsync()); //clean up await queueFactory.CloudQueueClient.GetQueueReference(queue.Name).DeleteIfExistsAsync(); }
public async Task WriteAnyMakerEventToQueue() { var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; var configurationContext = MakerDAOEventProcessingConfig.Create(PARTITION, out IdGenerator idGenerator); IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator); var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet); // queue components var queueFactory = new AzureSubscriberQueueFactory(azureStorageConnectionString); // load subscribers and event subscriptions var eventSubscriptionFactory = new EventSubscriptionFactory( web3, configurationRepository, queueFactory); List <IEventSubscription> eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION); // progress repo (dictates which block ranges to process next) // maintain separate progress per partition via a prefix var storageCloudSetup = new CloudTableSetup(azureStorageConnectionString, prefix: $"Partition{PARTITION}"); var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository(); //this ensures we only query the chain for events relating to this contract var makerAddressFilter = new NewFilterInput() { Address = new[] { MakerDAOEventProcessingConfig.MAKER_CONTRACT_ADDRESS } }; // load service var progressService = new BlockProgressService(web3, MIN_BLOCK_NUMBER, blockProgressRepo); var logProcessor = new BlockRangeLogsProcessor(web3, eventSubscriptions, makerAddressFilter); var batchProcessorService = new LogsProcessor(logProcessor, progressService, MAX_BLOCKS_PER_BATCH); // execute var blockRangesProcessed = new List <BlockRange?>(); try { for (var i = 0; i < 2; i++) // 2 batch iterations { var ctx = new System.Threading.CancellationTokenSource(); var rangeProcessed = await batchProcessorService.ProcessOnceAsync(ctx.Token); blockRangesProcessed.Add(rangeProcessed); // save event subscription state after each batch await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State)); } } finally { await ClearDown(configurationContext, storageCloudSetup, queueFactory); } var subscriptionState = await configurationRepository.EventSubscriptionStates.GetAsync(eventSubscriptions[0].Id); Assert.Equal(2, (int)subscriptionState.Values["HandlerInvocations"]); Assert.Equal(28, (int)subscriptionState.Values["EventsHandled"]); }
public async Task WebJobExample() { var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; string azureSearchKey = config["AzureSearchApiKey"]; var configurationContext = EventProcessingConfigMock.Create(PARTITION, out IdGenerator idGenerator); IEventProcessingConfigurationRepository configurationRepository = configurationContext.CreateMockRepository(idGenerator); var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby); // search components var searchService = new AzureSearchService(serviceName: AZURE_SEARCH_SERVICE_NAME, searchApiKey: azureSearchKey); var searchIndexFactory = new AzureSubscriberSearchIndexFactory(searchService); // queue components var queueFactory = new AzureSubscriberQueueFactory(azureStorageConnectionString); // subscriber repository var repositoryFactory = new AzureTablesSubscriberRepositoryFactory(azureStorageConnectionString); // load subscribers and event subscriptions var eventSubscriptionFactory = new EventSubscriptionFactory( web3, configurationRepository, queueFactory, searchIndexFactory, repositoryFactory); List <IEventSubscription> eventSubscriptions = await eventSubscriptionFactory.LoadAsync(PARTITION); // progress repo (dictates which block ranges to process next) // maintain separate progress per partition via a prefix var storageCloudSetup = new CloudTableSetup(azureStorageConnectionString, prefix: $"Partition{PARTITION}"); var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository(); // load service var progressService = new BlockProgressService(web3, MIN_BLOCK_NUMBER, blockProgressRepo); var logProcessor = new BlockRangeLogsProcessor(web3, eventSubscriptions); var batchProcessorService = new LogsProcessor(logProcessor, progressService, MAX_BLOCKS_PER_BATCH); // execute BlockRange?rangeProcessed; try { var ctx = new System.Threading.CancellationTokenSource(); rangeProcessed = await batchProcessorService.ProcessOnceAsync(ctx.Token); } finally { await ClearDown(configurationContext, storageCloudSetup, searchService, queueFactory, repositoryFactory); } // save event subscription state await configurationRepository.EventSubscriptionStates.UpsertAsync(eventSubscriptions.Select(s => s.State)); // assertions Assert.NotNull(rangeProcessed); Assert.Equal((ulong)10, rangeProcessed.Value.BlockCount); var subscriptionState1 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 1); // interested in transfers with contract queries and aggregations var subscriptionState2 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 2); // interested in transfers with simple aggregation var subscriptionState3 = configurationContext.GetEventSubscriptionState(eventSubscriptionId: 3); // interested in any event for a specific address Assert.Equal("4009000000002040652615", subscriptionState1.Values["RunningTotalForTransferValue"].ToString()); Assert.Equal((uint)19, subscriptionState2.Values["CurrentTransferCount"]); var txForSpecificAddress = (List <string>)subscriptionState3.Values["AllTransactionHashes"]; Assert.Equal("0x362bcbc78a5cc6156e8d24d95bee6b8f53d7821083940434d2191feba477ae0e", txForSpecificAddress[0]); Assert.Equal("0xe63e9422dedf84d0ce13f9f75ebfd86333ce917b2572925fbdd51b51caf89b77", txForSpecificAddress[1]); var blockNumbersForSpecificAddress = (List <HexBigInteger>)subscriptionState3.Values["AllBlockNumbers"]; Assert.Equal((BigInteger)4063362, blockNumbersForSpecificAddress[0].Value); Assert.Equal((BigInteger)4063362, blockNumbersForSpecificAddress[1].Value); }
public async Task QueueAllEventsForMakerDAOContract() { // Load config // - this will contain the secrets and connection strings we don't want to hard code var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; // Create a proxy for the blockchain var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet); // Create Queue Factory // - In this sample we're targetting Azure // - The factory communicates with Azure to create and get different queues var queueFactory = new AzureSubscriberQueueFactory(azureStorageConnectionString); // Create a Queue // - This is where we're going to put the matching event logs var queue = await queueFactory.GetOrCreateQueueAsync("makerdaoevents"); // Get the maker DAO contract abi // - from this we're able to match and decode the events in the contract var contractAbi = new ABIDeserialiser().DeserialiseContract(MAKER_DAO_ABI); // Create an event subscription for these events // - Passing in the maker dao address to ensure only logs with a matching address are processed // - There is an option to pass an implementation of IEventHandlerHistoryRepository in to the constructor // - This would record history for each event handler and is used to prevent duplication var eventSubscription = new EventSubscription(contractAbi.Events, new[] { MAKER_CONTRACT_ADDRESS }); // Assign the queue to the event subscription // - Matching events will be written to this queue // - By default a generic message is written to the queue // - The message contains the raw log (aka FilterLog), decoded event parameter values and event metadata // - Therefore the message schema is consistent across all messages sent to any queues // - However - should you require your own queue message schema the method below accepts a custom message mapper // - Ultimately the message is converted to json eventSubscription.AddQueueHandler(queue); // Azure storage setup // - this example reads and writes block progress to an Azure storage table // - to avoid collision with other samples we provide a prefix var storageCloudSetup = new CloudTableSetup(azureStorageConnectionString, prefix: $"makerdao"); // Create a progress repository // - It stores and retrieves the most recent block processed var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository(); // Create a progress service // - This uses the progress repo to dictate what blocks to process next // - The MIN_BLOCK_NUMBER dictates the starting point if the progress repo is empty or has fallen too far behind var progressService = new BlockProgressService(web3, MIN_BLOCK_NUMBER, blockProgressRepo); // Create a filter // - This is essentially the query that is sent to the chain when retrieving logs // - It is OPTIONAL - without it, all logs in the block range are requested // - The filter is invoked before any event subscriptions evaluate the logs // - The subscriptions are free to implement their own matching logic // - In this sample we're only interested in MakerDAO logs // - Therefore it makes sense to restrict the number of logs to retrieve from the chain var makerAddressFilter = new NewFilterInput() { Address = new[] { MAKER_CONTRACT_ADDRESS } }; // Create a log processor // - This uses the blockchainProxy to get the logs // - It sends each log to the event subscriptions to indicate if the logs matches the subscription criteria // - It then allocates matching logs to separate batches per event subscription var logProcessor = new BlockRangeLogsProcessor(web3, new[] { eventSubscription }, makerAddressFilter); // Create a batch log processor service // - It uses the progress service to calculates the block range to progress // - It then invokes the log processor - passing in the range to process // - It updates progress via the progress service var batchProcessorService = new LogsProcessor(logProcessor, progressService, MAX_BLOCKS_PER_BATCH); // execute try { // Optional cancellation token // - Useful for cancelling long running processing operations var ctx = new System.Threading.CancellationTokenSource(); // instruct the service to get and process the next range of blocks // when the rangeProcessed is null - it means there was nothing to process var rangeProcessed = await batchProcessorService.ProcessOnceAsync(ctx.Token); // ensure we have processed the expected number of events // the event subscription has state which can record running totals across many processing batches Assert.Equal(16, eventSubscription.State.GetInt("EventsHandled")); // get the message count from the queue Assert.Equal(16, await queue.GetApproxMessageCountAsync()); } finally { // delete any data from Azure await ClearDown(queue, storageCloudSetup, queueFactory); } }
public async Task WritingCustomMessagesToTheQueue() { // Load config // - this will contain the secrets and connection strings we don't want to hard code var config = TestConfiguration.LoadConfig(); string azureStorageConnectionString = config["AzureStorageConnectionString"]; // Create a proxy for the blockchain var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Mainnet); // Create Queue Factory // - In this sample we're targetting Azure // - The factory communicates with Azure to create and get different queues var queueFactory = new AzureSubscriberQueueFactory(azureStorageConnectionString); // Create a Queue // - This is where we're going to put the matching event logs var queue = await queueFactory.GetOrCreateQueueAsync("makerdaotransferscustom"); // Create an event subscription specifically for ERC20 Transfers // - Passing in the maker dao address to ensure only logs with a matching address are processed // - There is an option to pass an implementation of IEventHandlerHistoryRepository in to the constructor // - This would record history for each event handler and is used to prevent duplication var eventSubscription = new EventSubscription <TransferEventDto>( contractAddressesToMatch: new[] { MAKER_CONTRACT_ADDRESS }); // Create a mapper that will convert the DecodedEvent into a custom message we want on the queue // In this sample we're using a subscription that is specific to an EventDTO (EventSubscription<TransferEventDto>) // This ensures that the decodedEvent.DecodedEventDto property is populated during processing // ( If the subscription is not tied to an EventDTO the decodedEvent.DecodedEventDto property would be null // BUT we can still read the event arguments (aka parameters or topics) from the decodedEvent.Event property) var queueMessageMapper = new QueueMessageMapper((decodedEvent) => { return(new CustomQueueMessageForTransfers { BlockNumber = decodedEvent.Log.BlockNumber.Value.ToString(), TransactionHash = decodedEvent.Log.TransactionHash, LogIndex = decodedEvent.Log.LogIndex.Value.ToString(), Transfer = decodedEvent.DecodedEventDto as TransferEventDto }); }); // Assign the queue to the event subscription // - Matching events will be written to this queue // - Pass a custom mapper to create a suitable queue message // - Ultimately the message is converted to json eventSubscription.AddQueueHandler(queue, queueMessageMapper); // Azure storage setup // - this example reads and writes block progress to an Azure storage table // - to avoid collision with other samples we provide a prefix var storageCloudSetup = new CloudTableSetup(azureStorageConnectionString, prefix: $"makerdaotransferscustom"); // Create a progress repository // - It stores and retrieves the most recent block processed var blockProgressRepo = storageCloudSetup.CreateBlockProgressRepository(); // Create a progress service // - This uses the progress repo to dictate what blocks to process next // - The MIN_BLOCK_NUMBER dictates the starting point if the progress repo is empty or has fallen too far behind var progressService = new BlockProgressService(web3, MIN_BLOCK_NUMBER, blockProgressRepo); // Create a filter // - This is essentially the query that is sent to the chain when retrieving logs // - It is OPTIONAL - without it, all logs in the block range are requested // - The filter is invoked before any event subscriptions evaluate the logs // - The subscriptions are free to implement their own matching logic // - In this sample we're only interested in MakerDAO logs // - Therefore it makes sense to restrict the number of logs to retrieve from the chain var makerAddressFilter = new NewFilterInput() { Address = new[] { MAKER_CONTRACT_ADDRESS } }; // Create a log processor // - This uses the blockchainProxy to get the logs // - It sends each log to the event subscriptions to indicate if the logs matches the subscription criteria // - It then allocates matching logs to separate batches per event subscription var logProcessor = new BlockRangeLogsProcessor(web3, new[] { eventSubscription }, makerAddressFilter); // Create a batch log processor service // - It uses the progress service to calculates the block range to progress // - It then invokes the log processor - passing in the range to process // - It updates progress via the progress service var batchProcessorService = new LogsProcessor(logProcessor, progressService, MAX_BLOCKS_PER_BATCH); // execute try { // Optional cancellation token // - Useful for cancelling long running processing operations var ctx = new System.Threading.CancellationTokenSource(); // instruct the service to get and process the next range of blocks // when the rangeProcessed is null - it means there was nothing to process var rangeProcessed = await batchProcessorService.ProcessOnceAsync(ctx.Token); // ensure we have processed the expected number of events // the event subscription has state which can record running totals across many processing batches Assert.Equal(11, eventSubscription.State.GetInt("EventsHandled")); // get the message count from the queue Assert.Equal(11, await queue.GetApproxMessageCountAsync()); //A sample message body from the queue /* * {"BlockNumber":"7540010","TransactionHash":"0x8d58abc578f5e321f2e6b7c0637ccc60fbf62b39b120691cbf19ff201f5069b0","LogIndex":"132","Transfer":{"From":"0x296c61eaf5bea208bbabc65ae01c3bc5270fe386","To":"0x2a8f1a6af55b705b7daee0776d6f97302de2a839","Value":119928660890733235}} */ } finally { // delete any data from Azure await ClearDown(queue, storageCloudSetup, queueFactory); } }