public async Task <int> ExecuteAsync( CancellationToken cancellationToken = default(CancellationToken)) { TRepositoryFactory repoFactory = default(TRepositoryFactory); try { repoFactory = CreateRepositoryFactoryCallback(Configuration); BlockchainProcessor processor = CreateBlockChainProcessor(repoFactory); if (TargetConfiguration.ToBlock != null) { await processor.ExecuteAsync(TargetConfiguration.ToBlock.Value, cancellationToken, TargetConfiguration.FromBlock); } else { await processor.ExecuteAsync(cancellationToken, TargetConfiguration.FromBlock); } return(0); } catch (Exception ex) { log?.Error(ex); if (repoFactory != null) { DisposeAction?.Invoke(repoFactory); } return(1); } }
public async Task Run() { var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby); var transactionHandler = new SimpleTransactionHandler(); var handlers = new HandlerContainer { TransactionHandler = transactionHandler }; //only tx sent to this address var transactionFilter = TransactionFilter.To("0xc0e15e11306334258d61fee52a22d15e6c9c59e0"); var filter = new FilterContainer(transactionFilter); var blockProcessor = BlockProcessorFactory.Create( web3, handlers, filter, processTransactionsInParallel: false); var processingStrategy = new ProcessingStrategy(blockProcessor); var blockchainProcessor = new BlockchainProcessor(processingStrategy); var result = await blockchainProcessor.ExecuteAsync(2830143, 2830153); Assert.True(result); Assert.Equal(12, transactionHandler.TransactionsHandled.Count); Assert.Empty(transactionHandler.ContractCreationTransactionsHandled); }
public async Task Run() { var blockchainProxyService = new BlockchainProxyService("https://rinkeby.infura.io/v3/25e7b6dfc51040b3bfc0e47317d38f60"); var transactionHandler = new SimpleTransactionHandler(); var handlers = new HandlerContainer { TransactionHandler = transactionHandler }; //only tx sent to this address var transactionFilter = TransactionFilter.To("0xc0e15e11306334258d61fee52a22d15e6c9c59e0"); var filter = new FilterContainer(transactionFilter); var blockProcessor = BlockProcessorFactory.Create( blockchainProxyService, handlers, filter, processTransactionsInParallel: false); var processingStrategy = new ProcessingStrategy(blockProcessor); var blockchainProcessor = new BlockchainProcessor(processingStrategy); var result = await blockchainProcessor.ExecuteAsync(2830143, 2830153); Assert.True(result); Assert.Equal(12, transactionHandler.TransactionsHandled.Count); Assert.Empty(transactionHandler.ContractCreationTransactionsHandled); }
static void Main(string[] args) { // This particular sample should be run with the following CLI args // A random contract on the Rinkeby network was chosen as an example // --Blockchain rinkeby --FromBlock 3146650 System.Console.WriteLine("CLI args: " + string.Join(" ", args)); var appConfig = ConfigurationUtils.Build(args).AddConsoleLogging(); var targetBlockchain = BlockchainSourceConfigurationFactory.Get(appConfig); System.Console.WriteLine($"Target Node/Name (URL): {targetBlockchain.Name}, {targetBlockchain.BlockchainUrl}"); //only process transactions that created or called our contract var filters = new ContractSpecificFilterBuilder(ContractAddress).Filters; //for specific functions on our contract, output the name and input arg values var transactionRouter = new TransactionRouter(); transactionRouter.AddContractCreationHandler( new ContractCreationPrinter <GlitchGoonsItemConstructor>()); transactionRouter.AddTransactionHandler(new FunctionPrinter <BuyApprenticeFunction>()); transactionRouter.AddTransactionHandler(new FunctionPrinter <OpenChestFunction>()); //for specific events, output the values var transactionLogRouter = new TransactionLogRouter(); transactionLogRouter.AddHandler(new EventPrinter <TransferEvent>()); var handlers = new HandlerContainer() { TransactionHandler = transactionRouter, TransactionLogHandler = transactionLogRouter, }; var blockchainProxy = new BlockchainProxyService(targetBlockchain.BlockchainUrl); var blockProcessor = BlockProcessorFactory.Create( blockchainProxy, handlers, filters, processTransactionsInParallel: false); var strategy = new ProcessingStrategy(blockProcessor) { MinimumBlockConfirmations = 6 //wait for 6 block confirmations before processing block }; var blockchainProcessor = new BlockchainProcessor(strategy); blockchainProcessor.ExecuteAsync (targetBlockchain.FromBlock, targetBlockchain.ToBlock) .GetAwaiter().GetResult(); System.Console.WriteLine($"Contracts Created: {transactionRouter.ContractsCreated}"); System.Console.WriteLine($"Transactions Handled: {transactionRouter.TransactionsHandled}"); System.Console.ReadLine(); }
public void Fetch(BigInteger from, BigInteger to) { EthUtils.RunSync(() => processor.ExecuteAsync( startAtBlockNumberIfNotProcessed: from, toBlockNumber: (to != BigInteger.Zero) ? to : from, cancellationToken: cancellationToken) ); }
public static async Task <int> Execute( IBlockchainStoreRepositoryFactory repositoryFactory, BlockchainSourceConfiguration configuration, FilterContainer filterContainer = null, bool useGeth = false, ILog log = null) { IWeb3 web3 = useGeth ? new Web3Geth(configuration.BlockchainUrl) : new Web3.Web3(configuration.BlockchainUrl); using (var repositoryHandlerContext = new RepositoryHandlerContext(repositoryFactory)) { var blockProcessor = BlockProcessorFactory .Create( web3, repositoryHandlerContext.Handlers, filters: filterContainer, postVm: configuration.PostVm, processTransactionsInParallel: configuration.ProcessBlockTransactionsInParallel); var storageProcessingStrategy = new StorageProcessingStrategy( repositoryHandlerContext, blockProcessor) { MinimumBlockNumber = configuration.MinimumBlockNumber ?? 0, MinimumBlockConfirmations = configuration.MinimumBlockConfirmations ?? 0 }; var blockchainProcessor = new BlockchainProcessor(storageProcessingStrategy, log); var stopWatch = Stopwatch.StartNew(); var result = await blockchainProcessor.ExecuteAsync(configuration.FromBlock, configuration.ToBlock) .ConfigureAwait(false); System.Console.WriteLine("Duration: " + stopWatch.Elapsed); Debug.WriteLine($"Finished With Success: {result}"); System.Console.WriteLine("Finished. Success:" + result); System.Console.ReadLine(); return(result ? 0 : 1); } }
public static async Task <int> Execute( IBlockchainStoreRepositoryFactory repositoryFactory, BlockchainSourceConfiguration configuration, FilterContainer filterContainer = null, bool useGeth = false) { IWeb3Wrapper web3 = new Web3Wrapper( useGeth ? new Web3Geth(configuration.BlockchainUrl) : new Web3.Web3(configuration.BlockchainUrl)); using (_strategy = new PersistenceStrategy( repositoryFactory, filterContainer, minimumBlockNumber: configuration.MinimumBlockNumber ?? 0)) { var blockProcessor = new BlockProcessorFactory() .Create( web3, _strategy, configuration.PostVm, configuration.ProcessBlockTransactionsInParallel); var blockchainProcessor = new BlockchainProcessor(_strategy, blockProcessor); //this should not really be necessary //but without it, when the process is killed early, some csv records where not being flushed AppDomain.CurrentDomain.ProcessExit += (s, e) => { _strategy?.Dispose(); }; var stopWatch = Stopwatch.StartNew(); var result = await blockchainProcessor.ExecuteAsync(configuration.FromBlock, configuration.ToBlock) .ConfigureAwait(false); System.Console.WriteLine("Duration: " + stopWatch.Elapsed); Debug.WriteLine($"Finished With Success: {result}"); System.Console.WriteLine("Finished. Success:" + result); System.Console.ReadLine(); return(result ? 0 : 1); } }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { BlockchainProcessor processor = _web3.Processing.Blocks.CreateBlockProcessor(_blockProgressRepository, steps => { steps.BlockStep.AddProcessorHandler(_blockStorageStepHandler); steps.ContractCreationStep.AddProcessorHandler(_contractCreationStorageStepHandler); steps.FilterLogStep.AddProcessorHandler(_filterLogStorageStepHandler); steps.TransactionReceiptStep.AddProcessorHandler(_transactionReceiptStorageStepHandler); }, 1); try { await processor.ExecuteAsync(stoppingToken).AnyContext(); } catch (Exception e) { _logger.LogError(e, "{Class} -> {Method} -> Unexpected error with BlockchainProcessor", nameof(BlockIndexerHost), nameof(Run)); throw new LoggedException(e); } }
public async Task Run() { var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby); var transactionHandler = new SimpleTransactionHandler(); var handlers = new HandlerContainer { TransactionHandler = transactionHandler }; var blockProcessor = BlockProcessorFactory.Create( web3, handlers, processTransactionsInParallel: false); var processingStrategy = new ProcessingStrategy(blockProcessor); var blockchainProcessor = new BlockchainProcessor(processingStrategy); var result = await blockchainProcessor.ExecuteAsync(2830144, 2830145); Assert.True(result); Assert.Equal(20, transactionHandler.TransactionsHandled?.Count); Assert.Equal(5, transactionHandler.ContractCreationsHandled?.Count); }
public async Task Run() { var contractAddresses = new[] { "0xd2e474c616cc60fb95d8b5f86c1043fa4552611b" }; const ulong FromBlock = 4347; const ulong ToBlock = 4347; var web3geth = new Web3Geth(); var blockchainProxyService = new BlockchainProxyService(web3geth); var transactionHandler = new SimpleTransactionHandler(); var vmStackHandler = new VmStackHandler(); var contractHandler = new ContractHandler(); foreach (var contractAddress in contractAddresses) { contractHandler.ContractAddresses.Add(contractAddress); } var handlers = new HandlerContainer { ContractHandler = contractHandler, TransactionHandler = transactionHandler, TransactionVmStackHandler = vmStackHandler }; var blockProcessor = BlockProcessorFactory.Create( blockchainProxyService, handlers, processTransactionsInParallel: false, postVm: true); var processingStrategy = new ProcessingStrategy(blockProcessor) { }; var blockchainProcessor = new BlockchainProcessor(processingStrategy); var result = await blockchainProcessor.ExecuteAsync(FromBlock, ToBlock); Assert.True(result); }
public async Task Run() { var blockchainProxy = new BlockchainProxyService("https://rinkeby.infura.io/v3/25e7b6dfc51040b3bfc0e47317d38f60"); var transactionHandler = new SimpleTransactionHandler(); var handlers = new HandlerContainer { TransactionHandler = transactionHandler }; var blockProcessor = BlockProcessorFactory.Create( blockchainProxy, handlers, processTransactionsInParallel: false); var processingStrategy = new ProcessingStrategy(blockProcessor); var blockchainProcessor = new BlockchainProcessor(processingStrategy); var result = await blockchainProcessor.ExecuteAsync(2830144, 2830145); Assert.True(result); Assert.Equal(20, transactionHandler.TransactionsHandled?.Count); Assert.Equal(5, transactionHandler.ContractCreationsHandled?.Count); }
public static async Task Main(string[] args) { // the number of blocks in a range to process in one batch const int DefaultBlocksPerBatch = 10; const int RequestRetryWeight = 0; // see below for retry algorithm // ensures the processor does not process blocks considered unconfirmed const int MinimumBlockConfirmations = 6; // somewhere to put the logs var logs = new List <FilterLog>(); // the web3 object dictates the target network // it provides the basis for processing var web3 = new Web3("https://rinkeby.infura.io/v3/7238211010344719ad14a89db874158c"); // only logs matching this filter will be processed // in this example we are targetting logs from a specific contract var filter = new NewFilterInput() { Address = new[] { "0x9edcb9a9c4d34b5d6a082c86cb4f117a1394f831" } }; // for logs matching the filter apply our handler // this handler has an action which be invoked if the criteria matches // async overloads for the action and criteria are also available // this handler is for any log // for event specific handlers - there is EventLogProcessorHandler<TEventDto> var logProcessorHandler = new ProcessorHandler <FilterLog>( action: (log) => logs.Add(log), criteria: (log) => log.Removed == false); // the processor accepts multiple handlers // add our single handler to a list IEnumerable <ProcessorHandler <FilterLog> > logProcessorHandlers = new ProcessorHandler <FilterLog>[] { logProcessorHandler }; // the processor accepts an optional ILog // replace this with your own Common.Logging.Log implementation ILog logger = null; /* * === Internal Log Request Retry Algorithm === * If requests to retrieve logs from the client fails, subsequent requests will be retried based on this algorithm * It's aim is to throttle the number of blocks in the request range and avoid errors * The retry weight proportionately restricts the reduction in block range per retry * (pseudo code) * nextBlockRangeSize = numberOfBlocksPerRequest / (retryRequestNumber + 1) + (_retryWeight * retryRequestNumber); */ // load the components into a LogOrchestrator IBlockchainProcessingOrchestrator orchestrator = new LogOrchestrator( ethApi: web3.Eth, logProcessors: logProcessorHandlers, filterInput: filter, defaultNumberOfBlocksPerRequest: DefaultBlocksPerBatch, retryWeight: RequestRetryWeight); // create a progress repository // can dictate the starting block (depending on the execution arguments) // stores the last block progresssed // you can write your own or Nethereum provides multiple implementations // https://github.com/Nethereum/Nethereum.BlockchainStorage/ IBlockProgressRepository progressRepository = new InMemoryBlockchainProgressRepository(); // this strategy is applied while waiting for block confirmations // it will apply a wait to allow the chain to add new blocks // the wait duration is dependant on the number of retries // feel free to implement your own IWaitStrategy waitForBlockConfirmationsStrategy = new WaitStrategy(); // this retrieves the current block on the chain (the most recent block) // it determines the next block to process ensuring it is within the min block confirmations // in the scenario where processing is up to date with the chain (i.e. processing very recent blocks) // it will apply a wait until the minimum block confirmations is met ILastConfirmedBlockNumberService lastConfirmedBlockNumberService = new LastConfirmedBlockNumberService( web3.Eth.Blocks.GetBlockNumber, waitForBlockConfirmationsStrategy, MinimumBlockConfirmations); // instantiate the main processor var processor = new BlockchainProcessor( orchestrator, progressRepository, lastConfirmedBlockNumberService, logger); // if we need to stop the processor mid execution - call cancel on the token source var cancellationToken = new CancellationTokenSource(); //crawl the required block range await processor.ExecuteAsync( toBlockNumber : new BigInteger(3146690), cancellationToken : cancellationToken.Token, startAtBlockNumberIfNotProcessed : new BigInteger(3146684)); Console.WriteLine($"Expected 4 logs. Logs found: {logs.Count}."); }
public async Task ProcessingTransfers(bool useContractAddressFilter) { // Scenario: // We want to track transfers for thousands of contract addresses // for the test - these addresses are coming from a file // obviously you can replace this with your own implementation // 7590 contract addresses expected HashSet <string> contractAddresses = LoadContractAddresses(); // instantiate our web3 object var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby); // somewhere to store the balance of each account involved in a transfer // there's no opening balance implementation here - so some balances may become negative var balances = new Dictionary <string, BigInteger>(); // the action to handle each relevant transfer event // this is a trivial synchronous implementation (in memory balance management) // your own can action can be whatever you need it to be // this may involve persistence, lookups and Async calls etc // you can pass an async action if required (new Func<EventLog<TransferEventDTO>, Task> ...) var action = new Action <EventLog <TransferEventDTO> >(transferEventLog => { var from = transferEventLog.Event.From; var to = transferEventLog.Event.To; if (!balances.ContainsKey(from)) { balances.Add(from, 0); } if (!balances.ContainsKey(to)) { balances.Add(to, 0); } balances[from] = balances[from] - transferEventLog.Event.Value; balances[to] = balances[to] + transferEventLog.Event.Value; }); // we're using an in memory progress repo which tracks the blocks processed // for real use - replace this with your own persistent implementation of IBlockProgressRepository // this allows you to run the processor continually (if required) // and pick up where it left off after a restart var blockProgressRepository = new InMemoryBlockchainProgressRepository(); BlockchainProcessor processor = null; if (useContractAddressFilter == false) { // as we're handling thousands of contracts - // we're not defaulting to the more obvious CreateProcessorForContracts method // under the hood, that would result in a filter containing all of the addresses // that filter would be sent to the node on a GetLogs RPC request // that large filter may cause issues depending on the node/client // instead we're using an event specific filter to get all Transfers // the node will return transfers for any contract and // we'll do the extra address filtering in the criteria processor = web3.Processing.Logs.CreateProcessor <TransferEventDTO>( blockProgressRepository: blockProgressRepository, action: action, criteria: (transferEventLog) => contractAddresses.Contains(transferEventLog.Log.Address)); } else { // it may be worth experimenting with the alternative CreateProcessorForContracts method below // under the hood it creates a contract specific get logs filter containing the addresses // therefore you don't need any extra criteria for address checking // depending on the node and the number of contracts it may work better // against infura with a known block range and 7590 contracts - there is no real difference processor = web3.Processing.Logs.CreateProcessorForContracts <TransferEventDTO>( contractAddresses: contractAddresses.ToArray(), blockProgressRepository: blockProgressRepository, action: action); } //if we need to stop the processor mid execution - call cancel on the token source var cancellationTokenSource = new CancellationTokenSource(); //crawl the required block range await processor.ExecuteAsync( toBlockNumber : new BigInteger(3000000), cancellationToken : cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed : new BigInteger(2999500)); Assert.Equal(168, balances.Count); // ** CONTINUAL PROCESSING // kick off the processor and leave it running until cancellation // the progress repository will control which block to start from // it will keep processing until the cancellation token is cancelled // await processor.ExecuteAsync(cancellationTokenSource.Token); // ** OPTION: Start At Block Number If Not Processed // normally your progress repo dictates the starting block (last block processed + 1) // however you might want to override this behaviour // why?: // - you have not processed anything previously and wish to start at a specific block number // - the last block processed in your progress repo is too far behind and you wish to start at a more recent block // (the last block processed from your progress your repo will always win if it exceeds the "startAtBlockNumberIfNotProcessed" value) //await processor.ExecuteAsync(cancellationToken: cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed: blockToStartAt); }
public async Task ExecuteAsync(ILogger logger) { if (!_config.Enabled) { logger.LogInformation($"{nameof(ProcessPurchaseOrderEventLogs)} is not enabled - see app settings"); return; } const int RequestRetryWeight = 0; // see below for retry algorithm var web3 = new Web3.Web3(_eshopConfiguration.EthereumRpcUrl); var walletBuyerService = new WalletBuyerService(web3, _eshopConfiguration.BuyerWalletAddress); var purchasingContractAddress = await walletBuyerService.PurchasingQueryAsync(); var filter = new NewFilterInput { Address = new[] { purchasingContractAddress } }; ILog log = logger.ToILog(); EventLogProcessorHandler <PurchaseOrderCreatedLogEventDTO> poCreatedHandler = CreatePurchaseOrderCreatedHandler(logger); var logProcessorHandlers = new ProcessorHandler <FilterLog>[] { poCreatedHandler }; IBlockchainProcessingOrchestrator orchestrator = new LogOrchestrator( ethApi: web3.Eth, logProcessors: logProcessorHandlers, filterInput: filter, defaultNumberOfBlocksPerRequest: (int)_config.NumberOfBlocksPerBatch, retryWeight: RequestRetryWeight); IWaitStrategy waitForBlockConfirmationsStrategy = new WaitStrategy(); ILastConfirmedBlockNumberService lastConfirmedBlockNumberService = new LastConfirmedBlockNumberService( web3.Eth.Blocks.GetBlockNumber, waitForBlockConfirmationsStrategy, _config.MinimumBlockConfirmations, log); var processor = new BlockchainProcessor( orchestrator, BlockProgressRepository, lastConfirmedBlockNumberService); var cancellationToken = new CancellationTokenSource(_config.TimeoutMs); var currentBlockOnChain = await web3.Eth.Blocks.GetBlockNumber.SendRequestAsync(); var blockToProcessTo = currentBlockOnChain.Value - _config.MinimumBlockConfirmations; var lastBlockProcessed = await BlockProgressRepository.GetLastBlockNumberProcessedAsync(); var minStartingBlock = _config.GetMinimumStartingBlock(); logger.LogInformation( $"Processing logs. To Block: {blockToProcessTo}, Last Block Processed: {lastBlockProcessed ?? 0}, Min Block: {minStartingBlock}"); await processor.ExecuteAsync( toBlockNumber : blockToProcessTo, cancellationToken : cancellationToken.Token, startAtBlockNumberIfNotProcessed : minStartingBlock); }