private void ParseBlockTransactions(long x2, BlockWithTransactions ethBlock, Database.Block newBlock)
{
object myLock = new object();
//foreach (var ethtx in ethBlock.Transactions)
long x1 = x2;
Parallel.ForEach(ethBlock.Transactions, new ParallelOptions {
MaxDegreeOfParallelism = 5
}, (ethtx) =>
{
Database.Transaction tx = new Database.Transaction
{
Gas = ethtx.Gas.AsLong(),
Nonce = ethtx.Nonce.AsLong(),
GasPrice = ethtx.GasPrice.Value.ToString(),
TxHash = ethtx.TransactionHash.DeHash(),
TxSender = ethtx.From.DeHash(),
TxRecipient = ethtx.To.DeHash(),
TxValue = ethtx.Value.Value.ToString(),
TxIndex = (int)ethtx.TransactionIndex.AsLong(),
};
//get traces
try
{
var queryResult = GetTrace(ethtx.TransactionHash, _rpcUrls[(int)(x1 % _rpcUrls.Count)]);
tx.Traces = queryResult.Item1;
tx.Receipt = queryResult.Item2;
}
catch (Exception e)
{
SentrySdk.CaptureException(e);
//ignored
}
lock (myLock)
{
newBlock.Transactions.Add(tx);
}
});
}
public void RunImporter()
{
long lastTimeStamp = 0;
//get blocks
Policy pollyRetryPolicy = Policy
.Handle <Exception>()
.WaitAndRetry(new[]
{
TimeSpan.FromSeconds(1),
}, (exception, retryCount) =>
{
SentrySdk.CaptureMessage("Parity is overloaded. Will wait... " + retryCount.TotalSeconds);
Console.WriteLine("Parity is overloaded. Will wait... " + retryCount.TotalSeconds);
});
long startBlock = _db.GetLastBlock() + 1;
Web3 web3 = GetWeb3Client();
long maxBlock = web3.Eth.Blocks.GetBlockNumber.SendRequestAsync().Result.AsLong() - 20;
Console.WriteLine($"==> Running block #{startBlock} to #{maxBlock} <==");
Stopwatch sw = new Stopwatch();
DateTime startTime = DateTime.UtcNow;
for (long x = startBlock; x <= maxBlock; x = x + _batchSize)
{
sw.Restart();
long x2 = x;
Parallel.For(x, x + _batchSize, i =>
{
SentrySdk.WithScope(scope =>
{
scope.SetExtra("block-number", i);
BlockWithTransactions ethBlock;
try
{
SentrySdk.AddBreadcrumb($"Getting Block #{i}");
ethBlock = GetBlockWithTransactions(pollyRetryPolicy, web3, i);
if (ethBlock == null)
{
throw new Exception("Got null block from rpc");
}
}
catch (Exception e)
{
SentrySdk.CaptureException(new Exception("Unable to retrieve block", e));
return;
}
try
{
//get block fromn parity
SentrySdk.AddBreadcrumb($"Parsing Block #{i}");
Database.Block newBlock = new Database.Block
{
Author = ethBlock.Author.DeHash(),
BlockHash = ethBlock.BlockHash.DeHash(),
BlockNumber = ethBlock.Number.AsLong(),
Difficulty = double.Parse(ethBlock.Difficulty.AsString()),
GasLimit = ethBlock.GasLimit.AsLong(),
GasUsed = ethBlock.GasUsed.AsLong(),
Miner = ethBlock.Miner.DeHash(),
Size = ethBlock.Size.AsLong(),
Timestamp = (long)ethBlock.Timestamp.Value,
Transactions = new List <Database.Transaction>()
};
//newBlock.Difficulty = ethBlock.Difficulty.AsLong();
lastTimeStamp = newBlock.Timestamp;
SentrySdk.AddBreadcrumb($"Parsing Transactions for Block #{i}");
ParseBlockTransactions(x2, ethBlock, newBlock);
SentrySdk.AddBreadcrumb($"Inserting Block #{i} to database");
_db.Insert(newBlock);
Console.Write(".");
}
catch (Exception e)
{
scope.SetExtra("block-data", JsonConvert.SerializeObject(ethBlock));
SentrySdk.CaptureException(new Exception("Unable to process block", e));
}
});
});
sw.Stop();
double blocksPerSec = _batchSize / (sw.ElapsedMilliseconds / 1000.0);
double avgPerSec = (x - startBlock) / (DateTime.UtcNow - startTime).TotalSeconds;
double remainingSeconds = ((maxBlock - x) / avgPerSec);
if (remainingSeconds < 0 || remainingSeconds > 10e6)
{
remainingSeconds = 120;
}
TimeSpan time = TimeSpan.FromSeconds(remainingSeconds);
DateTimeOffset lastBlockDt = DateTimeOffset.FromUnixTimeSeconds(lastTimeStamp);
TimeSpan behindChain = DateTime.UtcNow - lastBlockDt.UtcDateTime;
Console.WriteLine(
$" => {x} blocks done - {blocksPerSec,6:F1} blk/s - AVG: {avgPerSec,6:F1} blk/s - ETA: {time.ToString(@"dd\.hh\:mm\:ss")} - {behindChain.ToString(@"dd\.hh\:mm\:ss")} behind chain =====");
}
Console.WriteLine("==> Batch done. Wait for new blocks. <==");
}
