private BlockchainProcessor CreateBlockChainProcessor(TRepositoryFactory repoFactory) { var web3 = new Web3.Web3(TargetConfiguration.BlockchainUrl); var steps = new BlockProcessingSteps(); steps.BlockStep.AddProcessorHandler((b) => { log.Info($"Processing block {b.Number.Value}, Tx Count: {b.Transactions.Length}"); return(Task.CompletedTask); }); steps.TransactionReceiptStep.AddProcessorHandler((tx) => { log.Info($"\tTransaction: Index: {tx.Transaction.TransactionIndex}, Hash: {tx.TransactionHash}"); return(Task.CompletedTask); }); steps.FilterLogStep.AddProcessorHandler((l) => { log.Info($"\t\tLog: Index: {l.LogIndex}"); return(Task.CompletedTask); }); var orchestrator = new BlockCrawlOrchestrator(web3.Eth, steps); orchestrator.ContractCreatedCrawlerStep.RetrieveCode = true; var lastConfirmedBlockNumberService = new LastConfirmedBlockNumberService( web3.Eth.Blocks.GetBlockNumber, TargetConfiguration.MinimumBlockConfirmations ?? LastConfirmedBlockNumberService.DEFAULT_BLOCK_CONFIRMATIONS, log); IBlockProgressRepository progressRepo = repoFactory.CreateBlockProgressRepository(); var processor = new BlockchainProcessor(orchestrator, progressRepo, lastConfirmedBlockNumberService, log); return(processor); }
public BlockchainProcessor CreateBlockProcessor( IBlockProgressRepository blockProgressRepository, Action <BlockProcessingSteps> stepsConfiguration, uint minimumBlockConfirmations, ILog log = null) { var processingSteps = new BlockProcessingSteps(); var orchestrator = new BlockCrawlOrchestrator(_ethApiContractService, processingSteps); var lastConfirmedBlockNumberService = new LastConfirmedBlockNumberService(_ethApiContractService.Blocks.GetBlockNumber, minimumBlockConfirmations); stepsConfiguration?.Invoke(processingSteps); return(new BlockchainProcessor(orchestrator, blockProgressRepository, lastConfirmedBlockNumberService, log)); }
//multi processor public BlockchainProcessor CreateProcessor( IEnumerable <ProcessorHandler <FilterLog> > logProcessors, uint minimumBlockConfirmations, NewFilterInput filter = null, IBlockProgressRepository blockProgressRepository = null, ILog log = null) { var orchestrator = new LogOrchestrator(_ethApiContractService, logProcessors, filter); var progressRepository = blockProgressRepository ?? new InMemoryBlockchainProgressRepository(); var lastConfirmedBlockNumberService = new LastConfirmedBlockNumberService(_ethApiContractService.Blocks.GetBlockNumber, minimumBlockConfirmations); return(new BlockchainProcessor(orchestrator, progressRepository, lastConfirmedBlockNumberService, log)); }
public BlockchainProcessor CreateBlockStorageProcessor( IBlockchainStoreRepositoryFactory blockchainStorageFactory, IBlockProgressRepository blockProgressRepository, uint minimumBlockConfirmations, Action <BlockProcessingSteps> configureSteps = null, ILog log = null) { var processingSteps = new BlockStorageProcessingSteps(blockchainStorageFactory); var orchestrator = new BlockCrawlOrchestrator(_ethApiContractService, processingSteps); if (blockProgressRepository == null && blockchainStorageFactory is IBlockProgressRepositoryFactory progressRepoFactory) { blockProgressRepository = progressRepoFactory.CreateBlockProgressRepository(); } blockProgressRepository = blockProgressRepository ?? new InMemoryBlockchainProgressRepository(); var lastConfirmedBlockNumberService = new LastConfirmedBlockNumberService(_ethApiContractService.Blocks.GetBlockNumber, minimumBlockConfirmations); configureSteps?.Invoke(processingSteps); return(new BlockchainProcessor(orchestrator, blockProgressRepository, lastConfirmedBlockNumberService, log)); }
public static async Task Main(string[] args) { // the number of blocks in a range to process in one batch const int DefaultBlocksPerBatch = 10; const int RequestRetryWeight = 0; // see below for retry algorithm // ensures the processor does not process blocks considered unconfirmed const int MinimumBlockConfirmations = 6; // somewhere to put the logs var logs = new List <FilterLog>(); // the web3 object dictates the target network // it provides the basis for processing var web3 = new Web3("https://rinkeby.infura.io/v3/7238211010344719ad14a89db874158c"); // only logs matching this filter will be processed // in this example we are targetting logs from a specific contract var filter = new NewFilterInput() { Address = new[] { "0x9edcb9a9c4d34b5d6a082c86cb4f117a1394f831" } }; // for logs matching the filter apply our handler // this handler has an action which be invoked if the criteria matches // async overloads for the action and criteria are also available // this handler is for any log // for event specific handlers - there is EventLogProcessorHandler<TEventDto> var logProcessorHandler = new ProcessorHandler <FilterLog>( action: (log) => logs.Add(log), criteria: (log) => log.Removed == false); // the processor accepts multiple handlers // add our single handler to a list IEnumerable <ProcessorHandler <FilterLog> > logProcessorHandlers = new ProcessorHandler <FilterLog>[] { logProcessorHandler }; // the processor accepts an optional ILog // replace this with your own Common.Logging.Log implementation ILog logger = null; /* * === Internal Log Request Retry Algorithm === * If requests to retrieve logs from the client fails, subsequent requests will be retried based on this algorithm * It's aim is to throttle the number of blocks in the request range and avoid errors * The retry weight proportionately restricts the reduction in block range per retry * (pseudo code) * nextBlockRangeSize = numberOfBlocksPerRequest / (retryRequestNumber + 1) + (_retryWeight * retryRequestNumber); */ // load the components into a LogOrchestrator IBlockchainProcessingOrchestrator orchestrator = new LogOrchestrator( ethApi: web3.Eth, logProcessors: logProcessorHandlers, filterInput: filter, defaultNumberOfBlocksPerRequest: DefaultBlocksPerBatch, retryWeight: RequestRetryWeight); // create a progress repository // can dictate the starting block (depending on the execution arguments) // stores the last block progresssed // you can write your own or Nethereum provides multiple implementations // https://github.com/Nethereum/Nethereum.BlockchainStorage/ IBlockProgressRepository progressRepository = new InMemoryBlockchainProgressRepository(); // this strategy is applied while waiting for block confirmations // it will apply a wait to allow the chain to add new blocks // the wait duration is dependant on the number of retries // feel free to implement your own IWaitStrategy waitForBlockConfirmationsStrategy = new WaitStrategy(); // this retrieves the current block on the chain (the most recent block) // it determines the next block to process ensuring it is within the min block confirmations // in the scenario where processing is up to date with the chain (i.e. processing very recent blocks) // it will apply a wait until the minimum block confirmations is met ILastConfirmedBlockNumberService lastConfirmedBlockNumberService = new LastConfirmedBlockNumberService( web3.Eth.Blocks.GetBlockNumber, waitForBlockConfirmationsStrategy, MinimumBlockConfirmations); // instantiate the main processor var processor = new BlockchainProcessor( orchestrator, progressRepository, lastConfirmedBlockNumberService, logger); // if we need to stop the processor mid execution - call cancel on the token source var cancellationToken = new CancellationTokenSource(); //crawl the required block range await processor.ExecuteAsync( toBlockNumber : new BigInteger(3146690), cancellationToken : cancellationToken.Token, startAtBlockNumberIfNotProcessed : new BigInteger(3146684)); Console.WriteLine($"Expected 4 logs. Logs found: {logs.Count}."); }
public async Task ExecuteAsync(ILogger logger) { if (!_config.Enabled) { logger.LogInformation($"{nameof(ProcessPurchaseOrderEventLogs)} is not enabled - see app settings"); return; } const int RequestRetryWeight = 0; // see below for retry algorithm var web3 = new Web3.Web3(_eshopConfiguration.EthereumRpcUrl); var walletBuyerService = new WalletBuyerService(web3, _eshopConfiguration.BuyerWalletAddress); var purchasingContractAddress = await walletBuyerService.PurchasingQueryAsync(); var filter = new NewFilterInput { Address = new[] { purchasingContractAddress } }; ILog log = logger.ToILog(); EventLogProcessorHandler <PurchaseOrderCreatedLogEventDTO> poCreatedHandler = CreatePurchaseOrderCreatedHandler(logger); var logProcessorHandlers = new ProcessorHandler <FilterLog>[] { poCreatedHandler }; IBlockchainProcessingOrchestrator orchestrator = new LogOrchestrator( ethApi: web3.Eth, logProcessors: logProcessorHandlers, filterInput: filter, defaultNumberOfBlocksPerRequest: (int)_config.NumberOfBlocksPerBatch, retryWeight: RequestRetryWeight); IWaitStrategy waitForBlockConfirmationsStrategy = new WaitStrategy(); ILastConfirmedBlockNumberService lastConfirmedBlockNumberService = new LastConfirmedBlockNumberService( web3.Eth.Blocks.GetBlockNumber, waitForBlockConfirmationsStrategy, _config.MinimumBlockConfirmations, log); var processor = new BlockchainProcessor( orchestrator, BlockProgressRepository, lastConfirmedBlockNumberService); var cancellationToken = new CancellationTokenSource(_config.TimeoutMs); var currentBlockOnChain = await web3.Eth.Blocks.GetBlockNumber.SendRequestAsync(); var blockToProcessTo = currentBlockOnChain.Value - _config.MinimumBlockConfirmations; var lastBlockProcessed = await BlockProgressRepository.GetLastBlockNumberProcessedAsync(); var minStartingBlock = _config.GetMinimumStartingBlock(); logger.LogInformation( $"Processing logs. To Block: {blockToProcessTo}, Last Block Processed: {lastBlockProcessed ?? 0}, Min Block: {minStartingBlock}"); await processor.ExecuteAsync( toBlockNumber : blockToProcessTo, cancellationToken : cancellationToken.Token, startAtBlockNumberIfNotProcessed : minStartingBlock); }