public async Task When_There_Is_Prior_Progress_A_Minimum_Starting_Block_Number_Will_Prevent_Processing_Earlier_Blocks()
{
var lastBlockProcessed = new BigInteger(5);
var minBlock = new BigInteger(10);
var progressRepository = new InMemoryBlockchainProgressRepository(lastBlockProcessed);
var mockRpcResponses = new BlockProcessingRpcMock(Web3Mock);
mockRpcResponses.AddToGetBlockNumberRequestQueue(100);
mockRpcResponses.SetupTransactionsWithReceipts(blockNumber: minBlock, numberOfTransactions: 2, logsPerTransaction: 2);
var cancellationTokenSource = new CancellationTokenSource();
var processedData = new ProcessedBlockchainData();
var blockProcessor = Web3.Processing.Blocks.CreateBlockProcessor(progressRepository, steps =>
{
//capture a block and then cancel
steps.BlockStep.AddSynchronousProcessorHandler((block) => {
processedData.Blocks.Add(block);
cancellationTokenSource.Cancel();
});
}
);
await blockProcessor.ExecuteAsync(cancellationTokenSource.Token, minBlock);
Assert.Single(processedData.Blocks); // one block processed
Assert.Equal(minBlock, processedData.Blocks[0].Number.Value); // should have been the next block
Assert.Equal(minBlock, await progressRepository.GetLastBlockNumberProcessedAsync()); // should have updated progress
}
public async Task When_There_Is_Prior_Progress_Processing_Should_Pick_Up_From_Where_It_Left_Off()
{
//pretend we have already completed block 1
var lastBlockProcessed = new BigInteger(1);
var nextBlockExpected = lastBlockProcessed + 1;
var progressRepository = new InMemoryBlockchainProgressRepository(lastBlockProcessed);
var mockRpcResponses = new BlockProcessingRpcMock(Web3Mock);
mockRpcResponses.AddToGetBlockNumberRequestQueue(100);
mockRpcResponses.SetupTransactionsWithReceipts(blockNumber: nextBlockExpected, numberOfTransactions: 2, logsPerTransaction: 2);
var processedData = new ProcessedBlockchainData();
var cancellationTokenSource = new CancellationTokenSource();
var blockProcessor = Web3.Processing.Blocks.CreateBlockProcessor(progressRepository, steps =>
{
//capture a block and then cancel
steps.BlockStep.AddSynchronousProcessorHandler((block) => {
processedData.Blocks.Add(block);
cancellationTokenSource.Cancel();
});
}
);
await blockProcessor.ExecuteAsync(cancellationTokenSource.Token);
Assert.Single(processedData.Blocks); // one block processed
Assert.Equal(nextBlockExpected, processedData.Blocks[0].Number.Value); // should have been the next block
Assert.Equal(nextBlockExpected, await progressRepository.GetLastBlockNumberProcessedAsync()); // should have updated progress
}
public async void ShouldGetTransferEventLogsUsingProcessorAndStoreThem()
{
var destinationAddress = "0x6C547791C3573c2093d81b919350DB1094707011";
//Using ropsten infura if wanted for only a tests
//var web3 = _ethereumClientIntegrationFixture.GetInfuraWeb3(InfuraNetwork.Ropsten);
var web3 = _ethereumClientIntegrationFixture.GetWeb3();
var erc20TokenDeployment = new ERC20TokenDeployment()
{
DecimalUnits = 18, TokenName = "TST", TokenSymbol = "TST", InitialAmount = Web3.Convert.ToWei(10000)
};
//Deploy our custom token
var tokenDeploymentReceipt = await ERC20TokenService.DeployContractAndWaitForReceiptAsync(web3, erc20TokenDeployment);
//Creating a new service
var tokenService = new ERC20TokenService(web3, tokenDeploymentReceipt.ContractAddress);
//using Web3.Convert.ToWei as it has 18 decimal places (default)
var transferReceipt1 = await tokenService.TransferRequestAndWaitForReceiptAsync(destinationAddress, Web3.Convert.ToWei(10, 18));
var transferReceipt2 = await tokenService.TransferRequestAndWaitForReceiptAsync(destinationAddress, Web3.Convert.ToWei(10, 18));
//We are storing in a database the logs
var storedMockedEvents = new List <EventLog <TransferEventDTO> >();
//storage action mock
Task StoreLogAsync(EventLog <TransferEventDTO> eventLog)
{
storedMockedEvents.Add(eventLog);
return(Task.CompletedTask);
}
//progress repository to restart processing (simple in memory one, use the other adapters for other storage possibilities)
var blockProgressRepository = new InMemoryBlockchainProgressRepository(transferReceipt1.BlockNumber.Value - 1);
//create our processor to retrieve transfers
//restrict the processor to Transfers for a specific contract address
var processor = web3.Processing.Logs.CreateProcessorForContract <TransferEventDTO>(
tokenService.ContractHandler.ContractAddress, //the contract to monitor
StoreLogAsync, //action to perform when a log is found
minimumBlockConfirmations: 0, // number of block confirmations to wait
blockProgressRepository: blockProgressRepository //repository to track the progress
);
//if we need to stop the processor mid execution - call cancel on the token
var cancellationToken = new CancellationToken();
//crawl the required block range
await processor.ExecuteAsync(
cancellationToken : cancellationToken,
toBlockNumber : transferReceipt2.BlockNumber.Value,
startAtBlockNumberIfNotProcessed : transferReceipt1.BlockNumber.Value);
Assert.Equal(2, storedMockedEvents.Count);
}
public async Task Will_Wait_For_Block_Confirmations_Before_Processing()
{
var blockLastProcessed = new BigInteger(100);
var nextBlock = blockLastProcessed + 1;
const uint MIN_CONFIRMATIONS = 12;
var progressRepository = new InMemoryBlockchainProgressRepository(blockLastProcessed);
var mockRpcResponses = new BlockProcessingRpcMock(Web3Mock);
//when first asked - pretend the current block is behind the required confirmations
mockRpcResponses.AddToGetBlockNumberRequestQueue(blockLastProcessed + MIN_CONFIRMATIONS);
//the next time return an incremented block which is under the confirmation limit
mockRpcResponses.AddToGetBlockNumberRequestQueue(nextBlock + MIN_CONFIRMATIONS);
mockRpcResponses.SetupTransactionsWithReceipts(blockNumber: nextBlock, numberOfTransactions: 2, logsPerTransaction: 2);
var cancellationTokenSource = new CancellationTokenSource();
var processedData = new ProcessedBlockchainData();
var blockProcessor = Web3.Processing.Blocks.CreateBlockProcessor(progressRepository, steps =>
{
//capture a block and then cancel
steps.BlockStep.AddSynchronousProcessorHandler((block) => {
processedData.Blocks.Add(block);
cancellationTokenSource.Cancel();
});
}
, MIN_CONFIRMATIONS);
await blockProcessor.ExecuteAsync(cancellationTokenSource.Token);
Assert.Single(processedData.Blocks); //should have processed a single block before cancellation
Assert.Equal(2, mockRpcResponses.BlockNumberRequestCount); //should have asked for latest block twice
Assert.Equal(nextBlock, processedData.Blocks[0].Number.Value); // should have handled the expected block
Assert.Equal(nextBlock, await progressRepository.GetLastBlockNumberProcessedAsync()); // should have updated progress
}
public static async Task Main(string[] args)
{
// the number of blocks in a range to process in one batch
const int DefaultBlocksPerBatch = 10;
const int RequestRetryWeight = 0; // see below for retry algorithm
// ensures the processor does not process blocks considered unconfirmed
const int MinimumBlockConfirmations = 6;
// somewhere to put the logs
var logs = new List <FilterLog>();
// the web3 object dictates the target network
// it provides the basis for processing
var web3 = new Web3("https://rinkeby.infura.io/v3/7238211010344719ad14a89db874158c");
// only logs matching this filter will be processed
// in this example we are targetting logs from a specific contract
var filter = new NewFilterInput()
{
Address = new[] { "0x9edcb9a9c4d34b5d6a082c86cb4f117a1394f831" }
};
// for logs matching the filter apply our handler
// this handler has an action which be invoked if the criteria matches
// async overloads for the action and criteria are also available
// this handler is for any log
// for event specific handlers - there is EventLogProcessorHandler<TEventDto>
var logProcessorHandler = new ProcessorHandler <FilterLog>(
action: (log) => logs.Add(log),
criteria: (log) => log.Removed == false);
// the processor accepts multiple handlers
// add our single handler to a list
IEnumerable <ProcessorHandler <FilterLog> > logProcessorHandlers = new ProcessorHandler <FilterLog>[] { logProcessorHandler };
// the processor accepts an optional ILog
// replace this with your own Common.Logging.Log implementation
ILog logger = null;
/*
* === Internal Log Request Retry Algorithm ===
* If requests to retrieve logs from the client fails, subsequent requests will be retried based on this algorithm
* It's aim is to throttle the number of blocks in the request range and avoid errors
* The retry weight proportionately restricts the reduction in block range per retry
* (pseudo code)
* nextBlockRangeSize = numberOfBlocksPerRequest / (retryRequestNumber + 1) + (_retryWeight * retryRequestNumber);
*/
// load the components into a LogOrchestrator
IBlockchainProcessingOrchestrator orchestrator = new LogOrchestrator(
ethApi: web3.Eth,
logProcessors: logProcessorHandlers,
filterInput: filter,
defaultNumberOfBlocksPerRequest: DefaultBlocksPerBatch,
retryWeight: RequestRetryWeight);
// create a progress repository
// can dictate the starting block (depending on the execution arguments)
// stores the last block progresssed
// you can write your own or Nethereum provides multiple implementations
// https://github.com/Nethereum/Nethereum.BlockchainStorage/
IBlockProgressRepository progressRepository = new InMemoryBlockchainProgressRepository();
// this strategy is applied while waiting for block confirmations
// it will apply a wait to allow the chain to add new blocks
// the wait duration is dependant on the number of retries
// feel free to implement your own
IWaitStrategy waitForBlockConfirmationsStrategy = new WaitStrategy();
// this retrieves the current block on the chain (the most recent block)
// it determines the next block to process ensuring it is within the min block confirmations
// in the scenario where processing is up to date with the chain (i.e. processing very recent blocks)
// it will apply a wait until the minimum block confirmations is met
ILastConfirmedBlockNumberService lastConfirmedBlockNumberService =
new LastConfirmedBlockNumberService(
web3.Eth.Blocks.GetBlockNumber, waitForBlockConfirmationsStrategy, MinimumBlockConfirmations);
// instantiate the main processor
var processor = new BlockchainProcessor(
orchestrator, progressRepository, lastConfirmedBlockNumberService, logger);
// if we need to stop the processor mid execution - call cancel on the token source
var cancellationToken = new CancellationTokenSource();
//crawl the required block range
await processor.ExecuteAsync(
toBlockNumber : new BigInteger(3146690),
cancellationToken : cancellationToken.Token,
startAtBlockNumberIfNotProcessed : new BigInteger(3146684));
Console.WriteLine($"Expected 4 logs. Logs found: {logs.Count}.");
}
public async Task ProcessingTransfers(bool useContractAddressFilter)
{
// Scenario:
// We want to track transfers for thousands of contract addresses
// for the test - these addresses are coming from a file
// obviously you can replace this with your own implementation
// 7590 contract addresses expected
HashSet <string> contractAddresses = LoadContractAddresses();
// instantiate our web3 object
var web3 = new Web3.Web3(TestConfiguration.BlockchainUrls.Infura.Rinkeby);
// somewhere to store the balance of each account involved in a transfer
// there's no opening balance implementation here - so some balances may become negative
var balances = new Dictionary <string, BigInteger>();
// the action to handle each relevant transfer event
// this is a trivial synchronous implementation (in memory balance management)
// your own can action can be whatever you need it to be
// this may involve persistence, lookups and Async calls etc
// you can pass an async action if required (new Func<EventLog<TransferEventDTO>, Task> ...)
var action = new Action <EventLog <TransferEventDTO> >(transferEventLog =>
{
var from = transferEventLog.Event.From;
var to = transferEventLog.Event.To;
if (!balances.ContainsKey(from))
{
balances.Add(from, 0);
}
if (!balances.ContainsKey(to))
{
balances.Add(to, 0);
}
balances[from] = balances[from] - transferEventLog.Event.Value;
balances[to] = balances[to] + transferEventLog.Event.Value;
});
// we're using an in memory progress repo which tracks the blocks processed
// for real use - replace this with your own persistent implementation of IBlockProgressRepository
// this allows you to run the processor continually (if required)
// and pick up where it left off after a restart
var blockProgressRepository = new InMemoryBlockchainProgressRepository();
BlockchainProcessor processor = null;
if (useContractAddressFilter == false)
{
// as we're handling thousands of contracts -
// we're not defaulting to the more obvious CreateProcessorForContracts method
// under the hood, that would result in a filter containing all of the addresses
// that filter would be sent to the node on a GetLogs RPC request
// that large filter may cause issues depending on the node/client
// instead we're using an event specific filter to get all Transfers
// the node will return transfers for any contract and
// we'll do the extra address filtering in the criteria
processor = web3.Processing.Logs.CreateProcessor <TransferEventDTO>(
blockProgressRepository: blockProgressRepository,
action: action,
criteria: (transferEventLog) => contractAddresses.Contains(transferEventLog.Log.Address));
}
else
{
// it may be worth experimenting with the alternative CreateProcessorForContracts method below
// under the hood it creates a contract specific get logs filter containing the addresses
// therefore you don't need any extra criteria for address checking
// depending on the node and the number of contracts it may work better
// against infura with a known block range and 7590 contracts - there is no real difference
processor = web3.Processing.Logs.CreateProcessorForContracts <TransferEventDTO>(
contractAddresses: contractAddresses.ToArray(),
blockProgressRepository: blockProgressRepository,
action: action);
}
//if we need to stop the processor mid execution - call cancel on the token source
var cancellationTokenSource = new CancellationTokenSource();
//crawl the required block range
await processor.ExecuteAsync(
toBlockNumber : new BigInteger(3000000),
cancellationToken : cancellationTokenSource.Token,
startAtBlockNumberIfNotProcessed : new BigInteger(2999500));
Assert.Equal(168, balances.Count);
// ** CONTINUAL PROCESSING
// kick off the processor and leave it running until cancellation
// the progress repository will control which block to start from
// it will keep processing until the cancellation token is cancelled
// await processor.ExecuteAsync(cancellationTokenSource.Token);
// ** OPTION: Start At Block Number If Not Processed
// normally your progress repo dictates the starting block (last block processed + 1)
// however you might want to override this behaviour
// why?:
// - you have not processed anything previously and wish to start at a specific block number
// - the last block processed in your progress repo is too far behind and you wish to start at a more recent block
// (the last block processed from your progress your repo will always win if it exceeds the "startAtBlockNumberIfNotProcessed" value)
//await processor.ExecuteAsync(cancellationToken: cancellationTokenSource.Token, startAtBlockNumberIfNotProcessed: blockToStartAt);
}