private void ParseBlockTransactions(long x2, BlockWithTransactions ethBlock, Database.Block newBlock) { object myLock = new object(); //foreach (var ethtx in ethBlock.Transactions) long x1 = x2; Parallel.ForEach(ethBlock.Transactions, new ParallelOptions { MaxDegreeOfParallelism = 5 }, (ethtx) => { Database.Transaction tx = new Database.Transaction { Gas = ethtx.Gas.AsLong(), Nonce = ethtx.Nonce.AsLong(), GasPrice = ethtx.GasPrice.Value.ToString(), TxHash = ethtx.TransactionHash.DeHash(), TxSender = ethtx.From.DeHash(), TxRecipient = ethtx.To.DeHash(), TxValue = ethtx.Value.Value.ToString(), TxIndex = (int)ethtx.TransactionIndex.AsLong(), }; //get traces try { var queryResult = GetTrace(ethtx.TransactionHash, _rpcUrls[(int)(x1 % _rpcUrls.Count)]); tx.Traces = queryResult.Item1; tx.Receipt = queryResult.Item2; } catch (Exception e) { SentrySdk.CaptureException(e); //ignored } lock (myLock) { newBlock.Transactions.Add(tx); } }); }
public void RunImporter() { long lastTimeStamp = 0; //get blocks Policy pollyRetryPolicy = Policy .Handle <Exception>() .WaitAndRetry(new[] { TimeSpan.FromSeconds(1), }, (exception, retryCount) => { SentrySdk.CaptureMessage("Parity is overloaded. Will wait... " + retryCount.TotalSeconds); Console.WriteLine("Parity is overloaded. Will wait... " + retryCount.TotalSeconds); }); long startBlock = _db.GetLastBlock() + 1; Web3 web3 = GetWeb3Client(); long maxBlock = web3.Eth.Blocks.GetBlockNumber.SendRequestAsync().Result.AsLong() - 20; Console.WriteLine($"==> Running block #{startBlock} to #{maxBlock} <=="); Stopwatch sw = new Stopwatch(); DateTime startTime = DateTime.UtcNow; for (long x = startBlock; x <= maxBlock; x = x + _batchSize) { sw.Restart(); long x2 = x; Parallel.For(x, x + _batchSize, i => { SentrySdk.WithScope(scope => { scope.SetExtra("block-number", i); BlockWithTransactions ethBlock; try { SentrySdk.AddBreadcrumb($"Getting Block #{i}"); ethBlock = GetBlockWithTransactions(pollyRetryPolicy, web3, i); if (ethBlock == null) { throw new Exception("Got null block from rpc"); } } catch (Exception e) { SentrySdk.CaptureException(new Exception("Unable to retrieve block", e)); return; } try { //get block fromn parity SentrySdk.AddBreadcrumb($"Parsing Block #{i}"); Database.Block newBlock = new Database.Block { Author = ethBlock.Author.DeHash(), BlockHash = ethBlock.BlockHash.DeHash(), BlockNumber = ethBlock.Number.AsLong(), Difficulty = double.Parse(ethBlock.Difficulty.AsString()), GasLimit = ethBlock.GasLimit.AsLong(), GasUsed = ethBlock.GasUsed.AsLong(), Miner = ethBlock.Miner.DeHash(), Size = ethBlock.Size.AsLong(), Timestamp = (long)ethBlock.Timestamp.Value, Transactions = new List <Database.Transaction>() }; //newBlock.Difficulty = ethBlock.Difficulty.AsLong(); lastTimeStamp = newBlock.Timestamp; SentrySdk.AddBreadcrumb($"Parsing Transactions for Block #{i}"); ParseBlockTransactions(x2, ethBlock, newBlock); SentrySdk.AddBreadcrumb($"Inserting Block #{i} to database"); _db.Insert(newBlock); Console.Write("."); } catch (Exception e) { scope.SetExtra("block-data", JsonConvert.SerializeObject(ethBlock)); SentrySdk.CaptureException(new Exception("Unable to process block", e)); } }); }); sw.Stop(); double blocksPerSec = _batchSize / (sw.ElapsedMilliseconds / 1000.0); double avgPerSec = (x - startBlock) / (DateTime.UtcNow - startTime).TotalSeconds; double remainingSeconds = ((maxBlock - x) / avgPerSec); if (remainingSeconds < 0 || remainingSeconds > 10e6) { remainingSeconds = 120; } TimeSpan time = TimeSpan.FromSeconds(remainingSeconds); DateTimeOffset lastBlockDt = DateTimeOffset.FromUnixTimeSeconds(lastTimeStamp); TimeSpan behindChain = DateTime.UtcNow - lastBlockDt.UtcDateTime; Console.WriteLine( $" => {x} blocks done - {blocksPerSec,6:F1} blk/s - AVG: {avgPerSec,6:F1} blk/s - ETA: {time.ToString(@"dd\.hh\:mm\:ss")} - {behindChain.ToString(@"dd\.hh\:mm\:ss")} behind chain ====="); } Console.WriteLine("==> Batch done. Wait for new blocks. <=="); }