Esempio n. 1
0
        public async Task When_There_Is_Prior_Progress_A_Minimum_Starting_Block_Number_Will_Prevent_Processing_Earlier_Blocks()
        {
            var lastBlockProcessed = new BigInteger(5);
            var minBlock           = new BigInteger(10);

            var progressRepository = new InMemoryBlockchainProgressRepository(lastBlockProcessed);

            var mockRpcResponses = new BlockProcessingRpcMock(Web3Mock);

            mockRpcResponses.AddToGetBlockNumberRequestQueue(100);
            mockRpcResponses.SetupTransactionsWithReceipts(blockNumber: minBlock, numberOfTransactions: 2, logsPerTransaction: 2);

            var cancellationTokenSource = new CancellationTokenSource();
            var processedData           = new ProcessedBlockchainData();

            var blockProcessor = Web3.Processing.Blocks.CreateBlockProcessor(progressRepository, steps =>
            {
                //capture a block and then cancel
                steps.BlockStep.AddSynchronousProcessorHandler((block) => {
                    processedData.Blocks.Add(block);
                    cancellationTokenSource.Cancel();
                });
            }
                                                                             );


            await blockProcessor.ExecuteAsync(cancellationTokenSource.Token, minBlock);

            Assert.Single(processedData.Blocks);                                                 // one block processed
            Assert.Equal(minBlock, processedData.Blocks[0].Number.Value);                        // should have been the next block
            Assert.Equal(minBlock, await progressRepository.GetLastBlockNumberProcessedAsync()); // should have updated progress
        }
Esempio n. 2
0
        public async Task When_There_Is_Prior_Progress_Processing_Should_Pick_Up_From_Where_It_Left_Off()
        {
            //pretend we have already completed block 1
            var lastBlockProcessed = new BigInteger(1);
            var nextBlockExpected  = lastBlockProcessed + 1;

            var progressRepository = new InMemoryBlockchainProgressRepository(lastBlockProcessed);

            var mockRpcResponses = new BlockProcessingRpcMock(Web3Mock);

            mockRpcResponses.AddToGetBlockNumberRequestQueue(100);
            mockRpcResponses.SetupTransactionsWithReceipts(blockNumber: nextBlockExpected, numberOfTransactions: 2, logsPerTransaction: 2);

            var processedData = new ProcessedBlockchainData();

            var cancellationTokenSource = new CancellationTokenSource();

            var blockProcessor = Web3.Processing.Blocks.CreateBlockProcessor(progressRepository, steps =>
            {
                //capture a block and then cancel
                steps.BlockStep.AddSynchronousProcessorHandler((block) => {
                    processedData.Blocks.Add(block);
                    cancellationTokenSource.Cancel();
                });
            }
                                                                             );

            await blockProcessor.ExecuteAsync(cancellationTokenSource.Token);

            Assert.Single(processedData.Blocks);                                                          // one block processed
            Assert.Equal(nextBlockExpected, processedData.Blocks[0].Number.Value);                        // should have been the next block
            Assert.Equal(nextBlockExpected, await progressRepository.GetLastBlockNumberProcessedAsync()); // should have updated progress
        }
Esempio n. 3
0
        public async void ShouldGetTransferEventLogsUsingProcessorAndStoreThem()
        {
            var destinationAddress = "0x6C547791C3573c2093d81b919350DB1094707011";
            //Using ropsten infura if wanted for only a tests
            //var web3 = _ethereumClientIntegrationFixture.GetInfuraWeb3(InfuraNetwork.Ropsten);
            var web3 = _ethereumClientIntegrationFixture.GetWeb3();

            var erc20TokenDeployment = new ERC20TokenDeployment()
            {
                DecimalUnits = 18, TokenName = "TST", TokenSymbol = "TST", InitialAmount = Web3.Convert.ToWei(10000)
            };

            //Deploy our custom token
            var tokenDeploymentReceipt = await ERC20TokenService.DeployContractAndWaitForReceiptAsync(web3, erc20TokenDeployment);

            //Creating a new service
            var tokenService = new ERC20TokenService(web3, tokenDeploymentReceipt.ContractAddress);

            //using Web3.Convert.ToWei as it has 18 decimal places (default)
            var transferReceipt1 = await tokenService.TransferRequestAndWaitForReceiptAsync(destinationAddress, Web3.Convert.ToWei(10, 18));

            var transferReceipt2 = await tokenService.TransferRequestAndWaitForReceiptAsync(destinationAddress, Web3.Convert.ToWei(10, 18));


            //We are storing in a database the logs
            var storedMockedEvents = new List <EventLog <TransferEventDTO> >();

            //storage action mock
            Task StoreLogAsync(EventLog <TransferEventDTO> eventLog)
            {
                storedMockedEvents.Add(eventLog);
                return(Task.CompletedTask);
            }

            //progress repository to restart processing (simple in memory one, use the other adapters for other storage possibilities)
            var blockProgressRepository = new InMemoryBlockchainProgressRepository(transferReceipt1.BlockNumber.Value - 1);

            //create our processor to retrieve transfers
            //restrict the processor to Transfers for a specific contract address
            var processor = web3.Processing.Logs.CreateProcessorForContract <TransferEventDTO>(
                tokenService.ContractHandler.ContractAddress,    //the contract to monitor
                StoreLogAsync,                                   //action to perform when a log is found
                minimumBlockConfirmations: 0,                    // number of block confirmations to wait
                blockProgressRepository: blockProgressRepository //repository to track the progress
                );

            //if we need to stop the processor mid execution - call cancel on the token
            var cancellationToken = new CancellationToken();

            //crawl the required block range
            await processor.ExecuteAsync(
                cancellationToken : cancellationToken,
                toBlockNumber : transferReceipt2.BlockNumber.Value,
                startAtBlockNumberIfNotProcessed : transferReceipt1.BlockNumber.Value);

            Assert.Equal(2, storedMockedEvents.Count);
        }
Esempio n. 4
0
        public async Task Will_Wait_For_Block_Confirmations_Before_Processing()
        {
            var        blockLastProcessed = new BigInteger(100);
            var        nextBlock          = blockLastProcessed + 1;
            const uint MIN_CONFIRMATIONS  = 12;

            var progressRepository = new InMemoryBlockchainProgressRepository(blockLastProcessed);

            var mockRpcResponses = new BlockProcessingRpcMock(Web3Mock);

            //when first asked - pretend the current block is behind the required confirmations
            mockRpcResponses.AddToGetBlockNumberRequestQueue(blockLastProcessed + MIN_CONFIRMATIONS);
            //the next time return an incremented block which is under the confirmation limit
            mockRpcResponses.AddToGetBlockNumberRequestQueue(nextBlock + MIN_CONFIRMATIONS);

            mockRpcResponses.SetupTransactionsWithReceipts(blockNumber: nextBlock, numberOfTransactions: 2, logsPerTransaction: 2);

            var cancellationTokenSource = new CancellationTokenSource();
            var processedData           = new ProcessedBlockchainData();

            var blockProcessor = Web3.Processing.Blocks.CreateBlockProcessor(progressRepository, steps =>
            {
                //capture a block and then cancel
                steps.BlockStep.AddSynchronousProcessorHandler((block) => {
                    processedData.Blocks.Add(block);
                    cancellationTokenSource.Cancel();
                });
            }
                                                                             , MIN_CONFIRMATIONS);


            await blockProcessor.ExecuteAsync(cancellationTokenSource.Token);

            Assert.Single(processedData.Blocks);                                                  //should have processed a single block before cancellation
            Assert.Equal(2, mockRpcResponses.BlockNumberRequestCount);                            //should have asked for latest block twice
            Assert.Equal(nextBlock, processedData.Blocks[0].Number.Value);                        // should have handled the expected block
            Assert.Equal(nextBlock, await progressRepository.GetLastBlockNumberProcessedAsync()); // should have updated progress
        }
Esempio n. 5
0
    public static async Task Main(string[] args)
    {
        // the number of blocks in a range to process in one batch
        const int DefaultBlocksPerBatch = 10;

        const int RequestRetryWeight = 0; // see below for retry algorithm

        // ensures the processor does not process blocks considered unconfirmed
        const int MinimumBlockConfirmations = 6;

        // somewhere to put the logs
        var logs = new List <FilterLog>();

        // the web3 object dictates the target network
        // it provides the basis for processing
        var web3 = new Web3("https://rinkeby.infura.io/v3/7238211010344719ad14a89db874158c");

        // only logs matching this filter will be processed
        // in this example we are targetting logs from a specific contract
        var filter = new NewFilterInput()
        {
            Address = new[] { "0x9edcb9a9c4d34b5d6a082c86cb4f117a1394f831" }
        };

        // for logs matching the filter apply our handler
        // this handler has an action which be invoked if the criteria matches
        // async overloads for the action and criteria are also available
        // this handler is for any log
        // for event specific handlers - there is EventLogProcessorHandler<TEventDto>
        var logProcessorHandler = new ProcessorHandler <FilterLog>(
            action: (log) => logs.Add(log),
            criteria: (log) => log.Removed == false);

        // the processor accepts multiple handlers
        // add our single handler to a list
        IEnumerable <ProcessorHandler <FilterLog> > logProcessorHandlers = new ProcessorHandler <FilterLog>[] { logProcessorHandler };

        // the processor accepts an optional ILog
        // replace this with your own Common.Logging.Log implementation
        ILog logger = null;

        /*
         * === Internal Log Request Retry Algorithm ===
         * If requests to retrieve logs from the client fails, subsequent requests will be retried based on this algorithm
         * It's aim is to throttle the number of blocks in the request range and avoid errors
         * The retry weight proportionately restricts the reduction in block range per retry
         * (pseudo code)
         * nextBlockRangeSize = numberOfBlocksPerRequest / (retryRequestNumber + 1) + (_retryWeight * retryRequestNumber);
         */

        // load the components into a LogOrchestrator
        IBlockchainProcessingOrchestrator orchestrator = new LogOrchestrator(
            ethApi: web3.Eth,
            logProcessors: logProcessorHandlers,
            filterInput: filter,
            defaultNumberOfBlocksPerRequest: DefaultBlocksPerBatch,
            retryWeight: RequestRetryWeight);

        // create a progress repository
        // can dictate the starting block (depending on the execution arguments)
        // stores the last block progresssed
        // you can write your own or Nethereum provides multiple implementations
        // https://github.com/Nethereum/Nethereum.BlockchainStorage/
        IBlockProgressRepository progressRepository = new InMemoryBlockchainProgressRepository();

        // this strategy is applied while waiting for block confirmations
        // it will apply a wait to allow the chain to add new blocks
        // the wait duration is dependant on the number of retries
        // feel free to implement your own
        IWaitStrategy waitForBlockConfirmationsStrategy = new WaitStrategy();

        // this retrieves the current block on the chain (the most recent block)
        // it determines the next block to process ensuring it is within the min block confirmations
        // in the scenario where processing is up to date with the chain (i.e. processing very recent blocks)
        // it will apply a wait until the minimum block confirmations is met
        ILastConfirmedBlockNumberService lastConfirmedBlockNumberService =
            new LastConfirmedBlockNumberService(
                web3.Eth.Blocks.GetBlockNumber, waitForBlockConfirmationsStrategy, MinimumBlockConfirmations);

        // instantiate the main processor
        var processor = new BlockchainProcessor(
            orchestrator, progressRepository, lastConfirmedBlockNumberService, logger);

        // if we need to stop the processor mid execution - call cancel on the token source
        var cancellationToken = new CancellationTokenSource();

        //crawl the required block range
        await processor.ExecuteAsync(
            toBlockNumber : new BigInteger(3146690),
            cancellationToken : cancellationToken.Token,
            startAtBlockNumberIfNotProcessed : new BigInteger(3146684));

        Console.WriteLine($"Expected 4 logs. Logs found: {logs.Count}.");
    }
        public async Task ProcessingTransfers(bool useContractAddressFilter)
        {
            // Scenario:
            // We want to track transfers for thousands of contract addresses

            // for the test - these addresses are coming from a file
            // obviously you can replace this with your own implementation
            // 7590 contract addresses expected
            HashSet <string> contractAddresses = LoadContractAddresses();

            // instantiate our web3 object
            var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby);

            // somewhere to store the balance of each account involved in a transfer
            // there's no opening balance implementation here - so some balances may become negative
            var balances = new Dictionary <string, BigInteger>();

            // the action to handle each relevant transfer event
            // this is a trivial synchronous implementation (in memory balance management)
            // your own can action can be whatever you need it to be
            // this may involve persistence, lookups and Async calls etc
            // you can pass an async action if required (new Func<EventLog<TransferEventDTO>, Task> ...)
            var action = new Action <EventLog <TransferEventDTO> >(transferEventLog =>
            {
                var from = transferEventLog.Event.From;
                var to   = transferEventLog.Event.To;

                if (!balances.ContainsKey(from))
                {
                    balances.Add(from, 0);
                }
                if (!balances.ContainsKey(to))
                {
                    balances.Add(to, 0);
                }

                balances[from] = balances[from] - transferEventLog.Event.Value;
                balances[to]   = balances[to] + transferEventLog.Event.Value;
            });

            // we're using an in memory progress repo which tracks the blocks processed
            // for real use - replace this with your own persistent implementation of IBlockProgressRepository
            // this allows you to run the processor continually (if required)
            // and pick up where it left off after a restart
            var blockProgressRepository = new InMemoryBlockchainProgressRepository();

            BlockchainProcessor processor = null;

            if (useContractAddressFilter == false)
            {
                // as we're handling thousands of contracts -
                // we're not defaulting to the more obvious CreateProcessorForContracts method
                // under the hood, that would result in a filter containing all of the addresses
                // that filter would be sent to the node on a GetLogs RPC request
                // that large filter may cause issues depending on the node/client
                // instead we're using an event specific filter to get all Transfers
                // the node will return transfers for any contract and
                // we'll do the extra address filtering in the criteria
                processor = web3.Processing.Logs.CreateProcessor <TransferEventDTO>(
                    blockProgressRepository: blockProgressRepository,
                    action: action,
                    criteria: (transferEventLog) => contractAddresses.Contains(transferEventLog.Log.Address));
            }
            else
            {
                // it may be worth experimenting with the alternative CreateProcessorForContracts method below
                // under the hood it creates a contract specific get logs filter containing the addresses
                // therefore you don't need any extra criteria for address checking
                // depending on the node and the number of contracts it may work better
                // against infura with a known block range and 7590 contracts - there is no real difference
                processor = web3.Processing.Logs.CreateProcessorForContracts <TransferEventDTO>(
                    contractAddresses: contractAddresses.ToArray(),
                    blockProgressRepository: blockProgressRepository,
                    action: action);
            }

            //if we need to stop the processor mid execution - call cancel on the token source
            var cancellationTokenSource = new CancellationTokenSource();

            //crawl the required block range
            await processor.ExecuteAsync(
                toBlockNumber : new BigInteger(3000000),
                cancellationToken : cancellationTokenSource.Token,
                startAtBlockNumberIfNotProcessed : new BigInteger(2999500));

            Assert.Equal(168, balances.Count);

            // ** CONTINUAL PROCESSING
            // kick off the processor and leave it running until cancellation
            // the progress repository will control which block to start from
            // it will keep processing until the cancellation token is cancelled
            // await processor.ExecuteAsync(cancellationTokenSource.Token);

            // ** OPTION: Start At Block Number If Not Processed
            // normally your progress repo dictates the starting block (last block processed + 1)
            // however you might want to override this behaviour
            // why?:
            // - you have not processed anything previously and wish to start at a specific block number
            // - the last block processed in your progress repo is too far behind and you wish to start at a more recent block
            // (the last block processed from your progress your repo will always win if it exceeds the "startAtBlockNumberIfNotProcessed" value)
            //await processor.ExecuteAsync(cancellationToken: cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed: blockToStartAt);
        }