public async Task <int> ExecuteAsync(
CancellationToken cancellationToken = default(CancellationToken))
{
TRepositoryFactory repoFactory = default(TRepositoryFactory);
try
{
repoFactory = CreateRepositoryFactoryCallback(Configuration);
BlockchainProcessor processor = CreateBlockChainProcessor(repoFactory);
if (TargetConfiguration.ToBlock != null)
{
await processor.ExecuteAsync(TargetConfiguration.ToBlock.Value, cancellationToken, TargetConfiguration.FromBlock);
}
else
{
await processor.ExecuteAsync(cancellationToken, TargetConfiguration.FromBlock);
}
return(0);
}
catch (Exception ex)
{
log?.Error(ex);
if (repoFactory != null)
{
DisposeAction?.Invoke(repoFactory);
}
return(1);
}
}
public async Task Run()
{
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby);
var transactionHandler = new SimpleTransactionHandler();
var handlers = new HandlerContainer {
TransactionHandler = transactionHandler
};
//only tx sent to this address
var transactionFilter = TransactionFilter.To("0xc0e15e11306334258d61fee52a22d15e6c9c59e0");
var filter = new FilterContainer(transactionFilter);
var blockProcessor = BlockProcessorFactory.Create(
web3,
handlers,
filter,
processTransactionsInParallel: false);
var processingStrategy = new ProcessingStrategy(blockProcessor);
var blockchainProcessor = new BlockchainProcessor(processingStrategy);
var result = await blockchainProcessor.ExecuteAsync(2830143, 2830153);
Assert.True(result);
Assert.Equal(12, transactionHandler.TransactionsHandled.Count);
Assert.Empty(transactionHandler.ContractCreationTransactionsHandled);
}
public async Task Run()
{
var blockchainProxyService = new BlockchainProxyService("https://rinkeby.infura.io/v3/25e7b6dfc51040b3bfc0e47317d38f60");
var transactionHandler = new SimpleTransactionHandler();
var handlers = new HandlerContainer {
TransactionHandler = transactionHandler
};
//only tx sent to this address
var transactionFilter = TransactionFilter.To("0xc0e15e11306334258d61fee52a22d15e6c9c59e0");
var filter = new FilterContainer(transactionFilter);
var blockProcessor = BlockProcessorFactory.Create(
blockchainProxyService,
handlers,
filter,
processTransactionsInParallel: false);
var processingStrategy = new ProcessingStrategy(blockProcessor);
var blockchainProcessor = new BlockchainProcessor(processingStrategy);
var result = await blockchainProcessor.ExecuteAsync(2830143, 2830153);
Assert.True(result);
Assert.Equal(12, transactionHandler.TransactionsHandled.Count);
Assert.Empty(transactionHandler.ContractCreationTransactionsHandled);
}
static void Main(string[] args)
{
// This particular sample should be run with the following CLI args
// A random contract on the Rinkeby network was chosen as an example
// --Blockchain rinkeby --FromBlock 3146650
System.Console.WriteLine("CLI args: " + string.Join(" ", args));
var appConfig = ConfigurationUtils.Build(args).AddConsoleLogging();
var targetBlockchain = BlockchainSourceConfigurationFactory.Get(appConfig);
System.Console.WriteLine($"Target Node/Name (URL): {targetBlockchain.Name}, {targetBlockchain.BlockchainUrl}");
//only process transactions that created or called our contract
var filters = new ContractSpecificFilterBuilder(ContractAddress).Filters;
//for specific functions on our contract, output the name and input arg values
var transactionRouter = new TransactionRouter();
transactionRouter.AddContractCreationHandler(
new ContractCreationPrinter <GlitchGoonsItemConstructor>());
transactionRouter.AddTransactionHandler(new FunctionPrinter <BuyApprenticeFunction>());
transactionRouter.AddTransactionHandler(new FunctionPrinter <OpenChestFunction>());
//for specific events, output the values
var transactionLogRouter = new TransactionLogRouter();
transactionLogRouter.AddHandler(new EventPrinter <TransferEvent>());
var handlers = new HandlerContainer()
{
TransactionHandler = transactionRouter,
TransactionLogHandler = transactionLogRouter,
};
var blockchainProxy = new BlockchainProxyService(targetBlockchain.BlockchainUrl);
var blockProcessor = BlockProcessorFactory.Create(
blockchainProxy,
handlers,
filters,
processTransactionsInParallel: false);
var strategy = new ProcessingStrategy(blockProcessor)
{
MinimumBlockConfirmations = 6 //wait for 6 block confirmations before processing block
};
var blockchainProcessor = new BlockchainProcessor(strategy);
blockchainProcessor.ExecuteAsync
(targetBlockchain.FromBlock, targetBlockchain.ToBlock)
.GetAwaiter().GetResult();
System.Console.WriteLine($"Contracts Created: {transactionRouter.ContractsCreated}");
System.Console.WriteLine($"Transactions Handled: {transactionRouter.TransactionsHandled}");
System.Console.ReadLine();
}
public void Fetch(BigInteger from, BigInteger to)
{
EthUtils.RunSync(() =>
processor.ExecuteAsync(
startAtBlockNumberIfNotProcessed: from,
toBlockNumber: (to != BigInteger.Zero) ? to : from,
cancellationToken: cancellationToken)
);
}
public static async Task <int> Execute(
IBlockchainStoreRepositoryFactory repositoryFactory,
BlockchainSourceConfiguration configuration,
FilterContainer filterContainer = null,
bool useGeth = false,
ILog log = null)
{
IWeb3 web3 =
useGeth
? new Web3Geth(configuration.BlockchainUrl)
: new Web3.Web3(configuration.BlockchainUrl);
using (var repositoryHandlerContext = new RepositoryHandlerContext(repositoryFactory))
{
var blockProcessor = BlockProcessorFactory
.Create(
web3,
repositoryHandlerContext.Handlers,
filters: filterContainer,
postVm: configuration.PostVm,
processTransactionsInParallel: configuration.ProcessBlockTransactionsInParallel);
var storageProcessingStrategy = new StorageProcessingStrategy(
repositoryHandlerContext, blockProcessor)
{
MinimumBlockNumber = configuration.MinimumBlockNumber ?? 0,
MinimumBlockConfirmations = configuration.MinimumBlockConfirmations ?? 0
};
var blockchainProcessor = new BlockchainProcessor(storageProcessingStrategy, log);
var stopWatch = Stopwatch.StartNew();
var result = await blockchainProcessor.ExecuteAsync(configuration.FromBlock, configuration.ToBlock)
.ConfigureAwait(false);
System.Console.WriteLine("Duration: " + stopWatch.Elapsed);
Debug.WriteLine($"Finished With Success: {result}");
System.Console.WriteLine("Finished. Success:" + result);
System.Console.ReadLine();
return(result ? 0 : 1);
}
}
public static async Task <int> Execute(
IBlockchainStoreRepositoryFactory repositoryFactory,
BlockchainSourceConfiguration configuration,
FilterContainer filterContainer = null,
bool useGeth = false)
{
IWeb3Wrapper web3 = new Web3Wrapper(
useGeth
? new Web3Geth(configuration.BlockchainUrl)
: new Web3.Web3(configuration.BlockchainUrl));
using (_strategy = new PersistenceStrategy(
repositoryFactory, filterContainer, minimumBlockNumber: configuration.MinimumBlockNumber ?? 0))
{
var blockProcessor = new BlockProcessorFactory()
.Create(
web3,
_strategy,
configuration.PostVm,
configuration.ProcessBlockTransactionsInParallel);
var blockchainProcessor = new BlockchainProcessor(_strategy, blockProcessor);
//this should not really be necessary
//but without it, when the process is killed early, some csv records where not being flushed
AppDomain.CurrentDomain.ProcessExit += (s, e) =>
{
_strategy?.Dispose();
};
var stopWatch = Stopwatch.StartNew();
var result = await blockchainProcessor.ExecuteAsync(configuration.FromBlock, configuration.ToBlock)
.ConfigureAwait(false);
System.Console.WriteLine("Duration: " + stopWatch.Elapsed);
Debug.WriteLine($"Finished With Success: {result}");
System.Console.WriteLine("Finished. Success:" + result);
System.Console.ReadLine();
return(result ? 0 : 1);
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
BlockchainProcessor processor = _web3.Processing.Blocks.CreateBlockProcessor(_blockProgressRepository, steps =>
{
steps.BlockStep.AddProcessorHandler(_blockStorageStepHandler);
steps.ContractCreationStep.AddProcessorHandler(_contractCreationStorageStepHandler);
steps.FilterLogStep.AddProcessorHandler(_filterLogStorageStepHandler);
steps.TransactionReceiptStep.AddProcessorHandler(_transactionReceiptStorageStepHandler);
}, 1);
try
{
await processor.ExecuteAsync(stoppingToken).AnyContext();
}
catch (Exception e)
{
_logger.LogError(e, "{Class} -> {Method} -> Unexpected error with BlockchainProcessor", nameof(BlockIndexerHost),
nameof(Run));
throw new LoggedException(e);
}
}
public async Task Run()
{
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby);
var transactionHandler = new SimpleTransactionHandler();
var handlers = new HandlerContainer {
TransactionHandler = transactionHandler
};
var blockProcessor = BlockProcessorFactory.Create(
web3,
handlers,
processTransactionsInParallel: false);
var processingStrategy = new ProcessingStrategy(blockProcessor);
var blockchainProcessor = new BlockchainProcessor(processingStrategy);
var result = await blockchainProcessor.ExecuteAsync(2830144, 2830145);
Assert.True(result);
Assert.Equal(20, transactionHandler.TransactionsHandled?.Count);
Assert.Equal(5, transactionHandler.ContractCreationsHandled?.Count);
}
public async Task Run()
{
var contractAddresses = new[]
{ "0xd2e474c616cc60fb95d8b5f86c1043fa4552611b" };
const ulong FromBlock = 4347;
const ulong ToBlock = 4347;
var web3geth = new Web3Geth();
var blockchainProxyService = new BlockchainProxyService(web3geth);
var transactionHandler = new SimpleTransactionHandler();
var vmStackHandler = new VmStackHandler();
var contractHandler = new ContractHandler();
foreach (var contractAddress in contractAddresses)
{
contractHandler.ContractAddresses.Add(contractAddress);
}
var handlers = new HandlerContainer {
ContractHandler = contractHandler, TransactionHandler = transactionHandler, TransactionVmStackHandler = vmStackHandler
};
var blockProcessor = BlockProcessorFactory.Create(
blockchainProxyService,
handlers,
processTransactionsInParallel: false,
postVm: true);
var processingStrategy = new ProcessingStrategy(blockProcessor)
{
};
var blockchainProcessor = new BlockchainProcessor(processingStrategy);
var result = await blockchainProcessor.ExecuteAsync(FromBlock, ToBlock);
Assert.True(result);
}
public async Task Run()
{
var blockchainProxy = new BlockchainProxyService("https://rinkeby.infura.io/v3/25e7b6dfc51040b3bfc0e47317d38f60");
var transactionHandler = new SimpleTransactionHandler();
var handlers = new HandlerContainer {
TransactionHandler = transactionHandler
};
var blockProcessor = BlockProcessorFactory.Create(
blockchainProxy,
handlers,
processTransactionsInParallel: false);
var processingStrategy = new ProcessingStrategy(blockProcessor);
var blockchainProcessor = new BlockchainProcessor(processingStrategy);
var result = await blockchainProcessor.ExecuteAsync(2830144, 2830145);
Assert.True(result);
Assert.Equal(20, transactionHandler.TransactionsHandled?.Count);
Assert.Equal(5, transactionHandler.ContractCreationsHandled?.Count);
}
public static async Task Main(string[] args)
{
// the number of blocks in a range to process in one batch
const int DefaultBlocksPerBatch = 10;
const int RequestRetryWeight = 0; // see below for retry algorithm
// ensures the processor does not process blocks considered unconfirmed
const int MinimumBlockConfirmations = 6;
// somewhere to put the logs
var logs = new List <FilterLog>();
// the web3 object dictates the target network
// it provides the basis for processing
var web3 = new Web3("https://rinkeby.infura.io/v3/7238211010344719ad14a89db874158c");
// only logs matching this filter will be processed
// in this example we are targetting logs from a specific contract
var filter = new NewFilterInput()
{
Address = new[] { "0x9edcb9a9c4d34b5d6a082c86cb4f117a1394f831" }
};
// for logs matching the filter apply our handler
// this handler has an action which be invoked if the criteria matches
// async overloads for the action and criteria are also available
// this handler is for any log
// for event specific handlers - there is EventLogProcessorHandler<TEventDto>
var logProcessorHandler = new ProcessorHandler <FilterLog>(
action: (log) => logs.Add(log),
criteria: (log) => log.Removed == false);
// the processor accepts multiple handlers
// add our single handler to a list
IEnumerable <ProcessorHandler <FilterLog> > logProcessorHandlers = new ProcessorHandler <FilterLog>[] { logProcessorHandler };
// the processor accepts an optional ILog
// replace this with your own Common.Logging.Log implementation
ILog logger = null;
/*
* === Internal Log Request Retry Algorithm ===
* If requests to retrieve logs from the client fails, subsequent requests will be retried based on this algorithm
* It's aim is to throttle the number of blocks in the request range and avoid errors
* The retry weight proportionately restricts the reduction in block range per retry
* (pseudo code)
* nextBlockRangeSize = numberOfBlocksPerRequest / (retryRequestNumber + 1) + (_retryWeight * retryRequestNumber);
*/
// load the components into a LogOrchestrator
IBlockchainProcessingOrchestrator orchestrator = new LogOrchestrator(
ethApi: web3.Eth,
logProcessors: logProcessorHandlers,
filterInput: filter,
defaultNumberOfBlocksPerRequest: DefaultBlocksPerBatch,
retryWeight: RequestRetryWeight);
// create a progress repository
// can dictate the starting block (depending on the execution arguments)
// stores the last block progresssed
// you can write your own or Nethereum provides multiple implementations
// https://github.com/Nethereum/Nethereum.BlockchainStorage/
IBlockProgressRepository progressRepository = new InMemoryBlockchainProgressRepository();
// this strategy is applied while waiting for block confirmations
// it will apply a wait to allow the chain to add new blocks
// the wait duration is dependant on the number of retries
// feel free to implement your own
IWaitStrategy waitForBlockConfirmationsStrategy = new WaitStrategy();
// this retrieves the current block on the chain (the most recent block)
// it determines the next block to process ensuring it is within the min block confirmations
// in the scenario where processing is up to date with the chain (i.e. processing very recent blocks)
// it will apply a wait until the minimum block confirmations is met
ILastConfirmedBlockNumberService lastConfirmedBlockNumberService =
new LastConfirmedBlockNumberService(
web3.Eth.Blocks.GetBlockNumber, waitForBlockConfirmationsStrategy, MinimumBlockConfirmations);
// instantiate the main processor
var processor = new BlockchainProcessor(
orchestrator, progressRepository, lastConfirmedBlockNumberService, logger);
// if we need to stop the processor mid execution - call cancel on the token source
var cancellationToken = new CancellationTokenSource();
//crawl the required block range
await processor.ExecuteAsync(
toBlockNumber : new BigInteger(3146690),
cancellationToken : cancellationToken.Token,
startAtBlockNumberIfNotProcessed : new BigInteger(3146684));
Console.WriteLine($"Expected 4 logs. Logs found: {logs.Count}.");
}
public async Task ProcessingTransfers(bool useContractAddressFilter)
{
// Scenario:
// We want to track transfers for thousands of contract addresses
// for the test - these addresses are coming from a file
// obviously you can replace this with your own implementation
// 7590 contract addresses expected
HashSet <string> contractAddresses = LoadContractAddresses();
// instantiate our web3 object
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby);
// somewhere to store the balance of each account involved in a transfer
// there's no opening balance implementation here - so some balances may become negative
var balances = new Dictionary <string, BigInteger>();
// the action to handle each relevant transfer event
// this is a trivial synchronous implementation (in memory balance management)
// your own can action can be whatever you need it to be
// this may involve persistence, lookups and Async calls etc
// you can pass an async action if required (new Func<EventLog<TransferEventDTO>, Task> ...)
var action = new Action <EventLog <TransferEventDTO> >(transferEventLog =>
{
var from = transferEventLog.Event.From;
var to = transferEventLog.Event.To;
if (!balances.ContainsKey(from))
{
balances.Add(from, 0);
}
if (!balances.ContainsKey(to))
{
balances.Add(to, 0);
}
balances[from] = balances[from] - transferEventLog.Event.Value;
balances[to] = balances[to] + transferEventLog.Event.Value;
});
// we're using an in memory progress repo which tracks the blocks processed
// for real use - replace this with your own persistent implementation of IBlockProgressRepository
// this allows you to run the processor continually (if required)
// and pick up where it left off after a restart
var blockProgressRepository = new InMemoryBlockchainProgressRepository();
BlockchainProcessor processor = null;
if (useContractAddressFilter == false)
{
// as we're handling thousands of contracts -
// we're not defaulting to the more obvious CreateProcessorForContracts method
// under the hood, that would result in a filter containing all of the addresses
// that filter would be sent to the node on a GetLogs RPC request
// that large filter may cause issues depending on the node/client
// instead we're using an event specific filter to get all Transfers
// the node will return transfers for any contract and
// we'll do the extra address filtering in the criteria
processor = web3.Processing.Logs.CreateProcessor <TransferEventDTO>(
blockProgressRepository: blockProgressRepository,
action: action,
criteria: (transferEventLog) => contractAddresses.Contains(transferEventLog.Log.Address));
}
else
{
// it may be worth experimenting with the alternative CreateProcessorForContracts method below
// under the hood it creates a contract specific get logs filter containing the addresses
// therefore you don't need any extra criteria for address checking
// depending on the node and the number of contracts it may work better
// against infura with a known block range and 7590 contracts - there is no real difference
processor = web3.Processing.Logs.CreateProcessorForContracts <TransferEventDTO>(
contractAddresses: contractAddresses.ToArray(),
blockProgressRepository: blockProgressRepository,
action: action);
}
//if we need to stop the processor mid execution - call cancel on the token source
var cancellationTokenSource = new CancellationTokenSource();
//crawl the required block range
await processor.ExecuteAsync(
toBlockNumber : new BigInteger(3000000),
cancellationToken : cancellationTokenSource.Token,
startAtBlockNumberIfNotProcessed : new BigInteger(2999500));
Assert.Equal(168, balances.Count);
// ** CONTINUAL PROCESSING
// kick off the processor and leave it running until cancellation
// the progress repository will control which block to start from
// it will keep processing until the cancellation token is cancelled
// await processor.ExecuteAsync(cancellationTokenSource.Token);
// ** OPTION: Start At Block Number If Not Processed
// normally your progress repo dictates the starting block (last block processed + 1)
// however you might want to override this behaviour
// why?:
// - you have not processed anything previously and wish to start at a specific block number
// - the last block processed in your progress repo is too far behind and you wish to start at a more recent block
// (the last block processed from your progress your repo will always win if it exceeds the "startAtBlockNumberIfNotProcessed" value)
//await processor.ExecuteAsync(cancellationToken: cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed: blockToStartAt);
}
public async Task ExecuteAsync(ILogger logger)
{
if (!_config.Enabled)
{
logger.LogInformation($"{nameof(ProcessPurchaseOrderEventLogs)} is not enabled - see app settings");
return;
}
const int RequestRetryWeight = 0; // see below for retry algorithm
var web3 = new Web3.Web3(_eshopConfiguration.EthereumRpcUrl);
var walletBuyerService = new WalletBuyerService(web3, _eshopConfiguration.BuyerWalletAddress);
var purchasingContractAddress = await walletBuyerService.PurchasingQueryAsync();
var filter = new NewFilterInput {
Address = new[] { purchasingContractAddress }
};
ILog log = logger.ToILog();
EventLogProcessorHandler <PurchaseOrderCreatedLogEventDTO> poCreatedHandler =
CreatePurchaseOrderCreatedHandler(logger);
var logProcessorHandlers = new ProcessorHandler <FilterLog>[]
{ poCreatedHandler };
IBlockchainProcessingOrchestrator orchestrator = new LogOrchestrator(
ethApi: web3.Eth,
logProcessors: logProcessorHandlers,
filterInput: filter,
defaultNumberOfBlocksPerRequest: (int)_config.NumberOfBlocksPerBatch,
retryWeight: RequestRetryWeight);
IWaitStrategy waitForBlockConfirmationsStrategy = new WaitStrategy();
ILastConfirmedBlockNumberService lastConfirmedBlockNumberService =
new LastConfirmedBlockNumberService(
web3.Eth.Blocks.GetBlockNumber,
waitForBlockConfirmationsStrategy,
_config.MinimumBlockConfirmations,
log);
var processor = new BlockchainProcessor(
orchestrator, BlockProgressRepository, lastConfirmedBlockNumberService);
var cancellationToken = new CancellationTokenSource(_config.TimeoutMs);
var currentBlockOnChain = await web3.Eth.Blocks.GetBlockNumber.SendRequestAsync();
var blockToProcessTo = currentBlockOnChain.Value - _config.MinimumBlockConfirmations;
var lastBlockProcessed = await BlockProgressRepository.GetLastBlockNumberProcessedAsync();
var minStartingBlock = _config.GetMinimumStartingBlock();
logger.LogInformation(
$"Processing logs. To Block: {blockToProcessTo}, Last Block Processed: {lastBlockProcessed ?? 0}, Min Block: {minStartingBlock}");
await processor.ExecuteAsync(
toBlockNumber : blockToProcessTo,
cancellationToken : cancellationToken.Token,
startAtBlockNumberIfNotProcessed : minStartingBlock);
}