private async Task <RocksDbProvider> GetRocksDbProvider(IDbConfig dbConfig, string basePath, bool useReceiptsDb)
{
RocksDbProvider debugRecorder = new RocksDbProvider(_context.LogManager);
await debugRecorder.Init(basePath, dbConfig, useReceiptsDb);
return(debugRecorder);
}
private async Task <RocksDbProvider> GetRocksDbProvider(IDbConfig dbConfig, string basePath, bool useReceiptsDb)
{
RocksDbProvider debugRecorder = new RocksDbProvider(_context.LogManager, _context.Config <INdmConfig>().Enabled);
ThisNodeInfo.AddInfo("DB location :", $"{basePath}");
await debugRecorder.Init(basePath, dbConfig, useReceiptsDb);
return(debugRecorder);
}
private async Task <RocksDbProvider> GetRocksDbProvider(IDbConfig dbConfig, string basePath, bool useReceiptsDb)
{
// bool addNdmDbs = _api.Config<INdmConfig>().Enabled;
IInitConfig initConfig = _api.Config <IInitConfig>();
RocksDbProvider debugRecorder = new RocksDbProvider(_api.LogManager, false, false);
ThisNodeInfo.AddInfo("DB location :", $"{basePath}");
await debugRecorder.Init(basePath, dbConfig, useReceiptsDb);
return(debugRecorder);
}
public async Task Execute()
{
ILogger logger = _context.LogManager.GetClassLogger();
/* sync */
IDbConfig dbConfig = _context.Config <IDbConfig>();
ISyncConfig syncConfig = _context.Config <ISyncConfig>();
IInitConfig initConfig = _context.Config <IInitConfig>();
foreach (PropertyInfo propertyInfo in typeof(IDbConfig).GetProperties())
{
if (logger.IsDebug)
{
logger.Debug($"DB {propertyInfo.Name}: {propertyInfo.GetValue(dbConfig)}");
}
}
if (initConfig.UseMemDb)
{
_context.DbProvider = new MemDbProvider();
}
else
{
RocksDbProvider rocksDbProvider = new RocksDbProvider(_context.LogManager);
await rocksDbProvider.Init(initConfig.BaseDbPath, dbConfig, initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync);
_context.DbProvider = rocksDbProvider;
}
if (syncConfig.BeamSync)
{
BeamSyncDbProvider beamSyncProvider = new BeamSyncDbProvider(_context.DbProvider, "processor DB", _context.LogManager);
_context.DbProvider = beamSyncProvider;
_context.NodeDataConsumer = beamSyncProvider.NodeDataConsumer;
}
// IDbProvider debugRecorder = new RocksDbProvider(Path.Combine(_context._initConfig.BaseDbPath, "debug"), dbConfig, _context._logManager, _context._initConfig.StoreTraces, _context._initConfig.StoreReceipts);
// _context._dbProvider = new RpcDbProvider(_context._jsonSerializer, new BasicJsonRpcClient(KnownRpcUris.Localhost, _context._jsonSerializer, _context._logManager), _context._logManager, debugRecorder);
// IDbProvider debugReader = new ReadOnlyDbProvider(new RocksDbProvider(Path.Combine(_context._initConfig.BaseDbPath, "debug"), dbConfig, _context._logManager, _context._initConfig.StoreTraces, _context._initConfig.StoreReceipts), false);
// _context._dbProvider = debugReader;
}
private static async Task RunBenchmarkBlocks()
{
Rlp.RegisterDecoders(typeof(ParityTraceDecoder).Assembly);
/* logging & instrumentation */
_logManager = new NLogManager("perfTest.logs.txt", null);
_logger = _logManager.GetClassLogger();
if (_logger.IsInfo)
{
_logger.Info("Deleting state DBs");
}
DeleteDb(FullStateDbPath);
DeleteDb(FullCodeDbPath);
DeleteDb(FullReceiptsDbPath);
DeleteDb(FullPendingTxsDbPath);
if (_logger.IsInfo)
{
_logger.Info("State DBs deleted");
}
/* load spec */
ChainSpecLoader loader = new ChainSpecLoader(new EthereumJsonSerializer());
string path = Path.Combine(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"chainspec", "ropsten.json"));
_logger.Info($"Loading ChainSpec from {path}");
ChainSpec chainSpec = loader.Load(File.ReadAllText(path));
_logger.Info($"ChainSpec loaded");
var specProvider = new ChainSpecBasedSpecProvider(chainSpec);
IRewardCalculator rewardCalculator = new RewardCalculator(specProvider);
var dbProvider = new RocksDbProvider(DbBasePath, DbConfig.Default, _logManager, true, true);
var stateDb = dbProvider.StateDb;
var codeDb = dbProvider.CodeDb;
var traceDb = dbProvider.TraceDb;
var blocksDb = dbProvider.BlocksDb;
var headersDb = dbProvider.HeadersDb;
var blockInfosDb = dbProvider.BlockInfosDb;
var receiptsDb = dbProvider.ReceiptsDb;
/* state & storage */
var stateProvider = new StateProvider(stateDb, codeDb, _logManager);
var storageProvider = new StorageProvider(stateDb, stateProvider, _logManager);
var ethereumSigner = new EthereumEcdsa(specProvider, _logManager);
var transactionPool = new TxPool(
NullTxStorage.Instance,
Timestamper.Default,
ethereumSigner,
specProvider,
new TxPoolConfig(),
stateProvider,
_logManager);
var blockInfoRepository = new ChainLevelInfoRepository(blockInfosDb);
var blockTree = new UnprocessedBlockTreeWrapper(new BlockTree(blocksDb, headersDb, blockInfosDb, blockInfoRepository, specProvider, transactionPool, _logManager));
var receiptStorage = new InMemoryReceiptStorage();
IBlockDataRecoveryStep recoveryStep = new TxSignaturesRecoveryStep(ethereumSigner, transactionPool, _logManager);
/* blockchain processing */
IList <IAdditionalBlockProcessor> blockProcessors = new List <IAdditionalBlockProcessor>();
var blockhashProvider = new BlockhashProvider(blockTree, LimboLogs.Instance);
var virtualMachine = new VirtualMachine(stateProvider, storageProvider, blockhashProvider, specProvider, _logManager);
var processor = new TransactionProcessor(specProvider, stateProvider, storageProvider, virtualMachine, _logManager);
ISealValidator sealValidator;
if (specProvider.ChainId == RopstenSpecProvider.Instance.ChainId)
{
var difficultyCalculator = new DifficultyCalculator(specProvider);
sealValidator = new EthashSealValidator(_logManager, difficultyCalculator, new Ethash(_logManager));
}
else if (chainSpec.SealEngineType == SealEngineType.Clique)
{
var snapshotManager = new SnapshotManager(CliqueConfig.Default, blocksDb, blockTree, ethereumSigner, _logManager);
sealValidator = new CliqueSealValidator(CliqueConfig.Default, snapshotManager, _logManager);
rewardCalculator = NoBlockRewards.Instance;
recoveryStep = new CompositeDataRecoveryStep(recoveryStep, new AuthorRecoveryStep(snapshotManager));
}
else if (chainSpec.SealEngineType == SealEngineType.AuRa)
{
var abiEncoder = new AbiEncoder();
var validatorProcessor = new AuRaAdditionalBlockProcessorFactory(dbProvider.StateDb, stateProvider, abiEncoder, processor, blockTree, receiptStorage, _logManager)
.CreateValidatorProcessor(chainSpec.AuRa.Validators);
sealValidator = new AuRaSealValidator(chainSpec.AuRa, new AuRaStepCalculator(chainSpec.AuRa.StepDuration, new Timestamper()), validatorProcessor, ethereumSigner, _logManager);
rewardCalculator = new AuRaRewardCalculator(chainSpec.AuRa, abiEncoder, processor);
blockProcessors.Add(validatorProcessor);
}
else
{
throw new NotSupportedException();
}
/* store & validation */
var headerValidator = new HeaderValidator(blockTree, sealValidator, specProvider, _logManager);
var ommersValidator = new OmmersValidator(blockTree, headerValidator, _logManager);
var transactionValidator = new TxValidator(chainSpec.ChainId);
var blockValidator = new BlockValidator(transactionValidator, headerValidator, ommersValidator, specProvider, _logManager);
/* blockchain processing */
var blockProcessor = new BlockProcessor(specProvider, blockValidator, rewardCalculator, processor, stateDb, codeDb, traceDb, stateProvider, storageProvider, transactionPool, receiptStorage, _logManager, blockProcessors);
var blockchainProcessor = new BlockchainProcessor(blockTree, blockProcessor, recoveryStep, _logManager, true, false);
if (chainSpec.SealEngineType == SealEngineType.AuRa)
{
stateProvider.CreateAccount(Address.Zero, UInt256.Zero);
storageProvider.Commit();
stateProvider.Commit(Homestead.Instance);
var finalizationManager = new AuRaBlockFinalizationManager(blockTree, blockInfoRepository, blockProcessor, blockProcessors.OfType <IAuRaValidator>().First(), _logManager);
}
foreach ((Address address, ChainSpecAllocation allocation) in chainSpec.Allocations)
{
stateProvider.CreateAccount(address, allocation.Balance);
if (allocation.Code != null)
{
Keccak codeHash = stateProvider.UpdateCode(allocation.Code);
stateProvider.UpdateCodeHash(address, codeHash, specProvider.GenesisSpec);
}
if (allocation.Constructor != null)
{
Transaction constructorTransaction = new Transaction(true)
{
SenderAddress = address,
Init = allocation.Constructor,
GasLimit = chainSpec.Genesis.GasLimit
};
processor.Execute(constructorTransaction, chainSpec.Genesis.Header, NullTxTracer.Instance);
}
}
_logger.Info($"Allocations configured, committing...");
stateProvider.Commit(specProvider.GenesisSpec);
_logger.Info($"Finalizing genesis...");
chainSpec.Genesis.Header.StateRoot = stateProvider.StateRoot;
chainSpec.Genesis.Header.Hash = BlockHeader.CalculateHash(chainSpec.Genesis.Header);
if (chainSpec.Genesis.Hash != blockTree.Genesis.Hash)
{
throw new Exception("Unexpected genesis hash");
}
_logger.Info($"Starting benchmark processor...");
/* start processing */
BigInteger totalGas = BigInteger.Zero;
Stopwatch stopwatch = new Stopwatch();
Block currentHead;
long maxMemory = 0;
blockTree.NewHeadBlock += (sender, args) =>
{
currentHead = args.Block;
if (currentHead.Number == 0)
{
return;
}
maxMemory = Math.Max(maxMemory, GC.GetTotalMemory(false));
totalGas += currentHead.GasUsed;
if ((BigInteger)args.Block.Number % 10000 == 9999)
{
stopwatch.Stop();
long ms = 1_000L * stopwatch.ElapsedTicks / Stopwatch.Frequency;
BigInteger number = args.Block.Number + 1;
_logger.Warn($"TOTAL after {number} (ms) : " + ms);
_logger.Warn($"TOTAL after {number} blocks/s : {(decimal) currentHead.Number / (ms / 1000m),5}");
_logger.Warn($"TOTAL after {number} Mgas/s : {((decimal) totalGas / 1000000) / (ms / 1000m),5}");
_logger.Warn($"TOTAL after {number} max mem : {maxMemory}");
_logger.Warn($"TOTAL after {number} GC (0/1/2) : {GC.CollectionCount(0)}/{GC.CollectionCount(1)}/{GC.CollectionCount(2)}");
_logger.Warn($"Is server GC {number} : {System.Runtime.GCSettings.IsServerGC}");
_logger.Warn($"GC latency mode {number} : {System.Runtime.GCSettings.LatencyMode}");
_logger.Warn($"TOTAL after {number} blocks DB reads : {Store.Metrics.BlocksDbReads}");
_logger.Warn($"TOTAL after {number} blocks DB writes : {Store.Metrics.BlocksDbWrites}");
_logger.Warn($"TOTAL after {number} infos DB reads : {Store.Metrics.BlockInfosDbReads}");
_logger.Warn($"TOTAL after {number} infos DB writes : {Store.Metrics.BlockInfosDbWrites}");
_logger.Warn($"TOTAL after {number} state tree reads : {Store.Metrics.StateTreeReads}");
_logger.Warn($"TOTAL after {number} state tree writes : {Store.Metrics.StateTreeWrites}");
_logger.Warn($"TOTAL after {number} state DB reads : {Store.Metrics.StateDbReads}");
_logger.Warn($"TOTAL after {number} state DB writes : {Store.Metrics.StateDbWrites}");
_logger.Warn($"TOTAL after {number} storage tree reads : {Store.Metrics.StorageTreeReads}");
_logger.Warn($"TOTAL after {number} storage tree writes : {Store.Metrics.StorageTreeWrites}");
_logger.Warn($"TOTAL after {number} tree node hash : {Store.Metrics.TreeNodeHashCalculations}");
_logger.Warn($"TOTAL after {number} tree node RLP decode : {Store.Metrics.TreeNodeRlpDecodings}");
_logger.Warn($"TOTAL after {number} tree node RLP encode : {Store.Metrics.TreeNodeRlpEncodings}");
_logger.Warn($"TOTAL after {number} code DB reads : {Store.Metrics.CodeDbReads}");
_logger.Warn($"TOTAL after {number} code DB writes : {Store.Metrics.CodeDbWrites}");
_logger.Warn($"TOTAL after {number} receipts DB reads : {Store.Metrics.ReceiptsDbReads}");
_logger.Warn($"TOTAL after {number} receipts DB writes : {Store.Metrics.ReceiptsDbWrites}");
_logger.Warn($"TOTAL after {number} other DB reads : {Store.Metrics.OtherDbReads}");
_logger.Warn($"TOTAL after {number} other DB writes : {Store.Metrics.OtherDbWrites}");
_logger.Warn($"TOTAL after {number} EVM exceptions : {Evm.Metrics.EvmExceptions}");
_logger.Warn($"TOTAL after {number} SLOAD opcodes : {Evm.Metrics.SloadOpcode}");
_logger.Warn($"TOTAL after {number} SSTORE opcodes : {Evm.Metrics.SstoreOpcode}");
_logger.Warn($"TOTAL after {number} EXP opcodes : {Evm.Metrics.ModExpOpcode}");
_logger.Warn($"TOTAL after {number} BLOCKHASH opcodes : {Evm.Metrics.BlockhashOpcode}");
_logger.Warn($"TOTAL after {number} EVM calls : {Evm.Metrics.Calls}");
_logger.Warn($"TOTAL after {number} RIPEMD Precompiles : {Evm.Metrics.Ripemd160Precompile}");
_logger.Warn($"TOTAL after {number} SHA256 Precompiles : {Evm.Metrics.Sha256Precompile}");
// disk space
stopwatch.Start();
}
};
bool isStarted = false;
TaskCompletionSource <object> completionSource = new TaskCompletionSource <object>();
blockTree.NewBestSuggestedBlock += (sender, args) =>
{
if (!isStarted)
{
blockchainProcessor.Process(blockTree.FindBlock(blockTree.Genesis.Hash, BlockTreeLookupOptions.RequireCanonical), ProcessingOptions.None, NullBlockTracer.Instance);
stopwatch.Start();
blockchainProcessor.Start();
isStarted = true;
}
if (args.Block.Number == BlocksToLoad)
{
completionSource.SetResult(null);
}
};
await Task.WhenAny(completionSource.Task, blockTree.LoadBlocksFromDb(CancellationToken.None, 0, 10000, BlocksToLoad));
await blockchainProcessor.StopAsync(true).ContinueWith(
t =>
{
if (t.IsFaulted)
{
_logger.Error("processing failed", t.Exception);
_logger.Error("inner", t.Exception.InnerException);
Console.ReadLine();
}
_logger.Info("Block processing completed.");
});
stopwatch.Stop();
Console.ReadLine();
}
private async Task InitBlockchain(InitParams initParams)
{
ChainSpec chainSpec = LoadChainSpec(initParams.ChainSpecPath);
/* spec */
// TODO: rebuild to use chainspec
ISpecProvider specProvider;
if (chainSpec.ChainId == RopstenSpecProvider.Instance.ChainId)
{
specProvider = RopstenSpecProvider.Instance;
}
else if (chainSpec.ChainId == MainNetSpecProvider.Instance.ChainId)
{
specProvider = MainNetSpecProvider.Instance;
}
else
{
throw new NotSupportedException($"Not yet tested, not yet supported ChainId {chainSpec.ChainId}");
}
var ethereumSigner = new EthereumSigner(specProvider, _logManager);
var transactionStore = new TransactionStore();
var sealEngine = ConfigureSealEngine(transactionStore, ethereumSigner, initParams);
/* sync */
var blocksDb = new DbOnTheRocks(Path.Combine(_dbBasePath, DbOnTheRocks.BlocksDbPath));
var blockInfosDb = new DbOnTheRocks(Path.Combine(_dbBasePath, DbOnTheRocks.BlockInfosDbPath));
var receiptsDb = new DbOnTheRocks(Path.Combine(_dbBasePath, DbOnTheRocks.ReceiptsDbPath));
/* blockchain */
var blockTree = new BlockTree(blocksDb, blockInfosDb, receiptsDb, specProvider, _logManager);
var difficultyCalculator = new DifficultyCalculator(specProvider);
/* validation */
var headerValidator = new HeaderValidator(difficultyCalculator, blockTree, sealEngine, specProvider, _logManager);
var ommersValidator = new OmmersValidator(blockTree, headerValidator, _logManager);
var txValidator = new TransactionValidator(new SignatureValidator(specProvider.ChainId));
var blockValidator = new BlockValidator(txValidator, headerValidator, ommersValidator, specProvider, _logManager);
/* state */
var dbProvider = new RocksDbProvider(_dbBasePath, _logManager);
var codeDb = new DbOnTheRocks(Path.Combine(_dbBasePath, DbOnTheRocks.CodeDbPath));
var stateDb = new DbOnTheRocks(Path.Combine(_dbBasePath, DbOnTheRocks.StateDbPath));
var stateTree = new StateTree(stateDb);
var stateProvider = new StateProvider(stateTree, codeDb, _logManager);
var storageProvider = new StorageProvider(dbProvider, stateProvider, _logManager);
/* blockchain processing */
var blockhashProvider = new BlockhashProvider(blockTree);
var virtualMachine = new VirtualMachine(stateProvider, storageProvider, blockhashProvider, _logManager);
var transactionProcessor = new TransactionProcessor(specProvider, stateProvider, storageProvider, virtualMachine, _tracer, _logManager);
var rewardCalculator = new RewardCalculator(specProvider);
var blockProcessor = new BlockProcessor(specProvider, blockValidator, rewardCalculator, transactionProcessor, dbProvider, stateProvider, storageProvider, transactionStore, _logManager);
_blockchainProcessor = new BlockchainProcessor(blockTree, sealEngine, transactionStore, difficultyCalculator, blockProcessor, ethereumSigner, _logManager);
// create shared objects between discovery and peer manager
_nodeFactory = new NodeFactory();
_nodeStatsProvider = new NodeStatsProvider(_configProvider);
var jsonSerializer = new JsonSerializer(_logManager);
var encrypter = new AesEncrypter(_configProvider, _logManager);
_keyStore = new FileKeyStore(_configProvider, jsonSerializer, encrypter, _cryptoRandom, _logManager);
//creating blockchain bridge
BlockchainBridge = new BlockchainBridge(ethereumSigner, stateProvider, _keyStore, blockTree, stateDb, transactionStore);
EthereumSigner = ethereumSigner;
_blockchainProcessor.Start();
LoadGenesisBlock(chainSpec, string.IsNullOrWhiteSpace(initParams.GenesisHash) ? null : new Keccak(initParams.GenesisHash), blockTree, stateProvider, specProvider);
await StartProcessing(blockTree, transactionStore, blockValidator, headerValidator, txValidator, initParams);
}
private async Task InitBlockchain()
{
ChainSpec chainSpec = LoadChainSpec(_initConfig.ChainSpecPath);
/* spec */
// TODO: rebuild to use chainspec
ISpecProvider specProvider;
if (chainSpec.ChainId == RopstenSpecProvider.Instance.ChainId)
{
specProvider = RopstenSpecProvider.Instance;
}
else if (chainSpec.ChainId == MainNetSpecProvider.Instance.ChainId)
{
specProvider = MainNetSpecProvider.Instance;
}
else
{
throw new NotSupportedException($"Not yet tested, not yet supported ChainId {chainSpec.ChainId}");
}
var ethereumSigner = new EthereumSigner(
specProvider,
_logManager);
var transactionStore = new TransactionStore();
var sealEngine = ConfigureSealEngine(
transactionStore,
ethereumSigner);
/* sync */
IDbConfig dbConfig = _configProvider.GetConfig <IDbConfig>();
foreach (PropertyInfo propertyInfo in typeof(IDbConfig).GetProperties())
{
_logger.Info($"DB {propertyInfo.Name}: {propertyInfo.GetValue(dbConfig)}");
}
var blocksDb = new DbOnTheRocks(
Path.Combine(_dbBasePath, DbOnTheRocks.BlocksDbPath),
dbConfig);
var blockInfosDb = new DbOnTheRocks(
Path.Combine(_dbBasePath, DbOnTheRocks.BlockInfosDbPath),
dbConfig);
var receiptsDb = new DbOnTheRocks(
Path.Combine(_dbBasePath, DbOnTheRocks.ReceiptsDbPath),
dbConfig);
/* blockchain */
_blockTree = new BlockTree(
blocksDb,
blockInfosDb,
receiptsDb,
specProvider,
_logManager);
var difficultyCalculator = new DifficultyCalculator(
specProvider);
/* validation */
var headerValidator = new HeaderValidator(
difficultyCalculator,
_blockTree,
sealEngine,
specProvider,
_logManager);
var ommersValidator = new OmmersValidator(
_blockTree,
headerValidator,
_logManager);
var txValidator = new TransactionValidator(
new SignatureValidator(specProvider.ChainId));
var blockValidator = new BlockValidator(
txValidator,
headerValidator,
ommersValidator,
specProvider,
_logManager);
/* state */
var dbProvider = new RocksDbProvider(
_dbBasePath,
_logManager,
dbConfig);
var stateDb = dbProvider.GetOrCreateStateDb();
var stateTree = new StateTree(stateDb);
var stateProvider = new StateProvider(
stateTree,
dbProvider.GetOrCreateCodeDb(),
_logManager);
var storageProvider = new StorageProvider(
dbProvider,
stateProvider,
_logManager);
/* blockchain processing */
var blockhashProvider = new BlockhashProvider(
_blockTree);
var virtualMachine = new VirtualMachine(
stateProvider,
storageProvider,
blockhashProvider,
_logManager);
var transactionProcessor = new TransactionProcessor(
specProvider,
stateProvider,
storageProvider,
virtualMachine,
_tracer,
_logManager);
var rewardCalculator = new RewardCalculator(
specProvider);
var blockProcessor = new BlockProcessor(
specProvider,
blockValidator,
rewardCalculator,
transactionProcessor,
dbProvider,
stateProvider,
storageProvider,
transactionStore,
_logManager);
_blockchainProcessor = new BlockchainProcessor(
_blockTree,
sealEngine,
transactionStore,
difficultyCalculator,
blockProcessor,
ethereumSigner,
_logManager, _perfService);
// create shared objects between discovery and peer manager
_nodeFactory = new NodeFactory();
_nodeStatsProvider = new NodeStatsProvider(_configProvider.GetConfig <IStatsConfig>(), _nodeFactory, _logManager);
var jsonSerializer = new JsonSerializer(
_logManager);
var encrypter = new AesEncrypter(
_configProvider,
_logManager);
_keyStore = new FileKeyStore(
_configProvider,
jsonSerializer,
encrypter,
_cryptoRandom,
_logManager);
//creating blockchain bridge
BlockchainBridge = new BlockchainBridge(
ethereumSigner,
stateProvider,
_keyStore,
_blockTree,
stateDb,
transactionStore);
EthereumSigner = ethereumSigner;
_blockchainProcessor.Start();
LoadGenesisBlock(chainSpec,
string.IsNullOrWhiteSpace(_initConfig.GenesisHash) ? null : new Keccak(_initConfig.GenesisHash),
_blockTree, stateProvider, specProvider);
#pragma warning disable 4014
LoadBlocksFromDb();
#pragma warning restore 4014
await InitializeNetwork(
transactionStore,
blockValidator,
headerValidator,
txValidator);
}
