public async Task When_There_Is_Prior_Progress_A_Minimum_Starting_Block_Number_Will_Prevent_Processing_Earlier_Blocks() { var lastBlockProcessed = new BigInteger(5); var minBlock = new BigInteger(10); var progressRepository = new InMemoryBlockchainProgressRepository(lastBlockProcessed); var mockRpcResponses = new BlockProcessingRpcMock(Web3Mock); mockRpcResponses.AddToGetBlockNumberRequestQueue(100); mockRpcResponses.SetupTransactionsWithReceipts(blockNumber: minBlock, numberOfTransactions: 2, logsPerTransaction: 2); var cancellationTokenSource = new CancellationTokenSource(); var processedData = new ProcessedBlockchainData(); var blockProcessor = Web3.Processing.Blocks.CreateBlockProcessor(progressRepository, steps => { //capture a block and then cancel steps.BlockStep.AddSynchronousProcessorHandler((block) => { processedData.Blocks.Add(block); cancellationTokenSource.Cancel(); }); } ); await blockProcessor.ExecuteAsync(cancellationTokenSource.Token, minBlock); Assert.Single(processedData.Blocks); // one block processed Assert.Equal(minBlock, processedData.Blocks[0].Number.Value); // should have been the next block Assert.Equal(minBlock, await progressRepository.GetLastBlockNumberProcessedAsync()); // should have updated progress }
public async Task When_There_Is_Prior_Progress_Processing_Should_Pick_Up_From_Where_It_Left_Off() { //pretend we have already completed block 1 var lastBlockProcessed = new BigInteger(1); var nextBlockExpected = lastBlockProcessed + 1; var progressRepository = new InMemoryBlockchainProgressRepository(lastBlockProcessed); var mockRpcResponses = new BlockProcessingRpcMock(Web3Mock); mockRpcResponses.AddToGetBlockNumberRequestQueue(100); mockRpcResponses.SetupTransactionsWithReceipts(blockNumber: nextBlockExpected, numberOfTransactions: 2, logsPerTransaction: 2); var processedData = new ProcessedBlockchainData(); var cancellationTokenSource = new CancellationTokenSource(); var blockProcessor = Web3.Processing.Blocks.CreateBlockProcessor(progressRepository, steps => { //capture a block and then cancel steps.BlockStep.AddSynchronousProcessorHandler((block) => { processedData.Blocks.Add(block); cancellationTokenSource.Cancel(); }); } ); await blockProcessor.ExecuteAsync(cancellationTokenSource.Token); Assert.Single(processedData.Blocks); // one block processed Assert.Equal(nextBlockExpected, processedData.Blocks[0].Number.Value); // should have been the next block Assert.Equal(nextBlockExpected, await progressRepository.GetLastBlockNumberProcessedAsync()); // should have updated progress }
public async void ShouldGetTransferEventLogsUsingProcessorAndStoreThem() { var destinationAddress = "0x6C547791C3573c2093d81b919350DB1094707011"; //Using ropsten infura if wanted for only a tests //var web3 = _ethereumClientIntegrationFixture.GetInfuraWeb3(InfuraNetwork.Ropsten); var web3 = _ethereumClientIntegrationFixture.GetWeb3(); var erc20TokenDeployment = new ERC20TokenDeployment() { DecimalUnits = 18, TokenName = "TST", TokenSymbol = "TST", InitialAmount = Web3.Convert.ToWei(10000) }; //Deploy our custom token var tokenDeploymentReceipt = await ERC20TokenService.DeployContractAndWaitForReceiptAsync(web3, erc20TokenDeployment); //Creating a new service var tokenService = new ERC20TokenService(web3, tokenDeploymentReceipt.ContractAddress); //using Web3.Convert.ToWei as it has 18 decimal places (default) var transferReceipt1 = await tokenService.TransferRequestAndWaitForReceiptAsync(destinationAddress, Web3.Convert.ToWei(10, 18)); var transferReceipt2 = await tokenService.TransferRequestAndWaitForReceiptAsync(destinationAddress, Web3.Convert.ToWei(10, 18)); //We are storing in a database the logs var storedMockedEvents = new List <EventLog <TransferEventDTO> >(); //storage action mock Task StoreLogAsync(EventLog <TransferEventDTO> eventLog) { storedMockedEvents.Add(eventLog); return(Task.CompletedTask); } //progress repository to restart processing (simple in memory one, use the other adapters for other storage possibilities) var blockProgressRepository = new InMemoryBlockchainProgressRepository(transferReceipt1.BlockNumber.Value - 1); //create our processor to retrieve transfers //restrict the processor to Transfers for a specific contract address var processor = web3.Processing.Logs.CreateProcessorForContract <TransferEventDTO>( tokenService.ContractHandler.ContractAddress, //the contract to monitor StoreLogAsync, //action to perform when a log is found minimumBlockConfirmations: 0, // number of block confirmations to wait blockProgressRepository: blockProgressRepository //repository to track the progress ); //if we need to stop the processor mid execution - call cancel on the token var cancellationToken = new CancellationToken(); //crawl the required block range await processor.ExecuteAsync( cancellationToken : cancellationToken, toBlockNumber : transferReceipt2.BlockNumber.Value, startAtBlockNumberIfNotProcessed : transferReceipt1.BlockNumber.Value); Assert.Equal(2, storedMockedEvents.Count); }
public async Task Will_Wait_For_Block_Confirmations_Before_Processing() { var blockLastProcessed = new BigInteger(100); var nextBlock = blockLastProcessed + 1; const uint MIN_CONFIRMATIONS = 12; var progressRepository = new InMemoryBlockchainProgressRepository(blockLastProcessed); var mockRpcResponses = new BlockProcessingRpcMock(Web3Mock); //when first asked - pretend the current block is behind the required confirmations mockRpcResponses.AddToGetBlockNumberRequestQueue(blockLastProcessed + MIN_CONFIRMATIONS); //the next time return an incremented block which is under the confirmation limit mockRpcResponses.AddToGetBlockNumberRequestQueue(nextBlock + MIN_CONFIRMATIONS); mockRpcResponses.SetupTransactionsWithReceipts(blockNumber: nextBlock, numberOfTransactions: 2, logsPerTransaction: 2); var cancellationTokenSource = new CancellationTokenSource(); var processedData = new ProcessedBlockchainData(); var blockProcessor = Web3.Processing.Blocks.CreateBlockProcessor(progressRepository, steps => { //capture a block and then cancel steps.BlockStep.AddSynchronousProcessorHandler((block) => { processedData.Blocks.Add(block); cancellationTokenSource.Cancel(); }); } , MIN_CONFIRMATIONS); await blockProcessor.ExecuteAsync(cancellationTokenSource.Token); Assert.Single(processedData.Blocks); //should have processed a single block before cancellation Assert.Equal(2, mockRpcResponses.BlockNumberRequestCount); //should have asked for latest block twice Assert.Equal(nextBlock, processedData.Blocks[0].Number.Value); // should have handled the expected block Assert.Equal(nextBlock, await progressRepository.GetLastBlockNumberProcessedAsync()); // should have updated progress }
public static async Task Main(string[] args) { // the number of blocks in a range to process in one batch const int DefaultBlocksPerBatch = 10; const int RequestRetryWeight = 0; // see below for retry algorithm // ensures the processor does not process blocks considered unconfirmed const int MinimumBlockConfirmations = 6; // somewhere to put the logs var logs = new List <FilterLog>(); // the web3 object dictates the target network // it provides the basis for processing var web3 = new Web3("https://rinkeby.infura.io/v3/7238211010344719ad14a89db874158c"); // only logs matching this filter will be processed // in this example we are targetting logs from a specific contract var filter = new NewFilterInput() { Address = new[] { "0x9edcb9a9c4d34b5d6a082c86cb4f117a1394f831" } }; // for logs matching the filter apply our handler // this handler has an action which be invoked if the criteria matches // async overloads for the action and criteria are also available // this handler is for any log // for event specific handlers - there is EventLogProcessorHandler<TEventDto> var logProcessorHandler = new ProcessorHandler <FilterLog>( action: (log) => logs.Add(log), criteria: (log) => log.Removed == false); // the processor accepts multiple handlers // add our single handler to a list IEnumerable <ProcessorHandler <FilterLog> > logProcessorHandlers = new ProcessorHandler <FilterLog>[] { logProcessorHandler }; // the processor accepts an optional ILog // replace this with your own Common.Logging.Log implementation ILog logger = null; /* * === Internal Log Request Retry Algorithm === * If requests to retrieve logs from the client fails, subsequent requests will be retried based on this algorithm * It's aim is to throttle the number of blocks in the request range and avoid errors * The retry weight proportionately restricts the reduction in block range per retry * (pseudo code) * nextBlockRangeSize = numberOfBlocksPerRequest / (retryRequestNumber + 1) + (_retryWeight * retryRequestNumber); */ // load the components into a LogOrchestrator IBlockchainProcessingOrchestrator orchestrator = new LogOrchestrator( ethApi: web3.Eth, logProcessors: logProcessorHandlers, filterInput: filter, defaultNumberOfBlocksPerRequest: DefaultBlocksPerBatch, retryWeight: RequestRetryWeight); // create a progress repository // can dictate the starting block (depending on the execution arguments) // stores the last block progresssed // you can write your own or Nethereum provides multiple implementations // https://github.com/Nethereum/Nethereum.BlockchainStorage/ IBlockProgressRepository progressRepository = new InMemoryBlockchainProgressRepository(); // this strategy is applied while waiting for block confirmations // it will apply a wait to allow the chain to add new blocks // the wait duration is dependant on the number of retries // feel free to implement your own IWaitStrategy waitForBlockConfirmationsStrategy = new WaitStrategy(); // this retrieves the current block on the chain (the most recent block) // it determines the next block to process ensuring it is within the min block confirmations // in the scenario where processing is up to date with the chain (i.e. processing very recent blocks) // it will apply a wait until the minimum block confirmations is met ILastConfirmedBlockNumberService lastConfirmedBlockNumberService = new LastConfirmedBlockNumberService( web3.Eth.Blocks.GetBlockNumber, waitForBlockConfirmationsStrategy, MinimumBlockConfirmations); // instantiate the main processor var processor = new BlockchainProcessor( orchestrator, progressRepository, lastConfirmedBlockNumberService, logger); // if we need to stop the processor mid execution - call cancel on the token source var cancellationToken = new CancellationTokenSource(); //crawl the required block range await processor.ExecuteAsync( toBlockNumber : new BigInteger(3146690), cancellationToken : cancellationToken.Token, startAtBlockNumberIfNotProcessed : new BigInteger(3146684)); Console.WriteLine($"Expected 4 logs. Logs found: {logs.Count}."); }
public async Task ProcessingTransfers(bool useContractAddressFilter) { // Scenario: // We want to track transfers for thousands of contract addresses // for the test - these addresses are coming from a file // obviously you can replace this with your own implementation // 7590 contract addresses expected HashSet <string> contractAddresses = LoadContractAddresses(); // instantiate our web3 object var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby); // somewhere to store the balance of each account involved in a transfer // there's no opening balance implementation here - so some balances may become negative var balances = new Dictionary <string, BigInteger>(); // the action to handle each relevant transfer event // this is a trivial synchronous implementation (in memory balance management) // your own can action can be whatever you need it to be // this may involve persistence, lookups and Async calls etc // you can pass an async action if required (new Func<EventLog<TransferEventDTO>, Task> ...) var action = new Action <EventLog <TransferEventDTO> >(transferEventLog => { var from = transferEventLog.Event.From; var to = transferEventLog.Event.To; if (!balances.ContainsKey(from)) { balances.Add(from, 0); } if (!balances.ContainsKey(to)) { balances.Add(to, 0); } balances[from] = balances[from] - transferEventLog.Event.Value; balances[to] = balances[to] + transferEventLog.Event.Value; }); // we're using an in memory progress repo which tracks the blocks processed // for real use - replace this with your own persistent implementation of IBlockProgressRepository // this allows you to run the processor continually (if required) // and pick up where it left off after a restart var blockProgressRepository = new InMemoryBlockchainProgressRepository(); BlockchainProcessor processor = null; if (useContractAddressFilter == false) { // as we're handling thousands of contracts - // we're not defaulting to the more obvious CreateProcessorForContracts method // under the hood, that would result in a filter containing all of the addresses // that filter would be sent to the node on a GetLogs RPC request // that large filter may cause issues depending on the node/client // instead we're using an event specific filter to get all Transfers // the node will return transfers for any contract and // we'll do the extra address filtering in the criteria processor = web3.Processing.Logs.CreateProcessor <TransferEventDTO>( blockProgressRepository: blockProgressRepository, action: action, criteria: (transferEventLog) => contractAddresses.Contains(transferEventLog.Log.Address)); } else { // it may be worth experimenting with the alternative CreateProcessorForContracts method below // under the hood it creates a contract specific get logs filter containing the addresses // therefore you don't need any extra criteria for address checking // depending on the node and the number of contracts it may work better // against infura with a known block range and 7590 contracts - there is no real difference processor = web3.Processing.Logs.CreateProcessorForContracts <TransferEventDTO>( contractAddresses: contractAddresses.ToArray(), blockProgressRepository: blockProgressRepository, action: action); } //if we need to stop the processor mid execution - call cancel on the token source var cancellationTokenSource = new CancellationTokenSource(); //crawl the required block range await processor.ExecuteAsync( toBlockNumber : new BigInteger(3000000), cancellationToken : cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed : new BigInteger(2999500)); Assert.Equal(168, balances.Count); // ** CONTINUAL PROCESSING // kick off the processor and leave it running until cancellation // the progress repository will control which block to start from // it will keep processing until the cancellation token is cancelled // await processor.ExecuteAsync(cancellationTokenSource.Token); // ** OPTION: Start At Block Number If Not Processed // normally your progress repo dictates the starting block (last block processed + 1) // however you might want to override this behaviour // why?: // - you have not processed anything previously and wish to start at a specific block number // - the last block processed in your progress repo is too far behind and you wish to start at a more recent block // (the last block processed from your progress your repo will always win if it exceeds the "startAtBlockNumberIfNotProcessed" value) //await processor.ExecuteAsync(cancellationToken: cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed: blockToStartAt); }