private async Task <RocksDbProvider> GetRocksDbProvider(IDbConfig dbConfig, string basePath, bool useReceiptsDb) { RocksDbProvider debugRecorder = new RocksDbProvider(_context.LogManager); await debugRecorder.Init(basePath, dbConfig, useReceiptsDb); return(debugRecorder); }
private async Task <RocksDbProvider> GetRocksDbProvider(IDbConfig dbConfig, string basePath, bool useReceiptsDb) { RocksDbProvider debugRecorder = new RocksDbProvider(_context.LogManager, _context.Config <INdmConfig>().Enabled); ThisNodeInfo.AddInfo("DB location :", $"{basePath}"); await debugRecorder.Init(basePath, dbConfig, useReceiptsDb); return(debugRecorder); }
private async Task <RocksDbProvider> GetRocksDbProvider(IDbConfig dbConfig, string basePath, bool useReceiptsDb) { // bool addNdmDbs = _api.Config<INdmConfig>().Enabled; IInitConfig initConfig = _api.Config <IInitConfig>(); RocksDbProvider debugRecorder = new RocksDbProvider(_api.LogManager, false, false); ThisNodeInfo.AddInfo("DB location :", $"{basePath}"); await debugRecorder.Init(basePath, dbConfig, useReceiptsDb); return(debugRecorder); }
public async Task Execute() { ILogger logger = _context.LogManager.GetClassLogger(); /* sync */ IDbConfig dbConfig = _context.Config <IDbConfig>(); ISyncConfig syncConfig = _context.Config <ISyncConfig>(); IInitConfig initConfig = _context.Config <IInitConfig>(); foreach (PropertyInfo propertyInfo in typeof(IDbConfig).GetProperties()) { if (logger.IsDebug) { logger.Debug($"DB {propertyInfo.Name}: {propertyInfo.GetValue(dbConfig)}"); } } if (initConfig.UseMemDb) { _context.DbProvider = new MemDbProvider(); } else { RocksDbProvider rocksDbProvider = new RocksDbProvider(_context.LogManager); await rocksDbProvider.Init(initConfig.BaseDbPath, dbConfig, initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync); _context.DbProvider = rocksDbProvider; } if (syncConfig.BeamSync) { BeamSyncDbProvider beamSyncProvider = new BeamSyncDbProvider(_context.DbProvider, "processor DB", _context.LogManager); _context.DbProvider = beamSyncProvider; _context.NodeDataConsumer = beamSyncProvider.NodeDataConsumer; } // IDbProvider debugRecorder = new RocksDbProvider(Path.Combine(_context._initConfig.BaseDbPath, "debug"), dbConfig, _context._logManager, _context._initConfig.StoreTraces, _context._initConfig.StoreReceipts); // _context._dbProvider = new RpcDbProvider(_context._jsonSerializer, new BasicJsonRpcClient(KnownRpcUris.Localhost, _context._jsonSerializer, _context._logManager), _context._logManager, debugRecorder); // IDbProvider debugReader = new ReadOnlyDbProvider(new RocksDbProvider(Path.Combine(_context._initConfig.BaseDbPath, "debug"), dbConfig, _context._logManager, _context._initConfig.StoreTraces, _context._initConfig.StoreReceipts), false); // _context._dbProvider = debugReader; }
private static async Task RunBenchmarkBlocks() { Rlp.RegisterDecoders(typeof(ParityTraceDecoder).Assembly); /* logging & instrumentation */ _logManager = new NLogManager("perfTest.logs.txt", null); _logger = _logManager.GetClassLogger(); if (_logger.IsInfo) { _logger.Info("Deleting state DBs"); } DeleteDb(FullStateDbPath); DeleteDb(FullCodeDbPath); DeleteDb(FullReceiptsDbPath); DeleteDb(FullPendingTxsDbPath); if (_logger.IsInfo) { _logger.Info("State DBs deleted"); } /* load spec */ ChainSpecLoader loader = new ChainSpecLoader(new EthereumJsonSerializer()); string path = Path.Combine(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"chainspec", "ropsten.json")); _logger.Info($"Loading ChainSpec from {path}"); ChainSpec chainSpec = loader.Load(File.ReadAllText(path)); _logger.Info($"ChainSpec loaded"); var specProvider = new ChainSpecBasedSpecProvider(chainSpec); IRewardCalculator rewardCalculator = new RewardCalculator(specProvider); var dbProvider = new RocksDbProvider(DbBasePath, DbConfig.Default, _logManager, true, true); var stateDb = dbProvider.StateDb; var codeDb = dbProvider.CodeDb; var traceDb = dbProvider.TraceDb; var blocksDb = dbProvider.BlocksDb; var headersDb = dbProvider.HeadersDb; var blockInfosDb = dbProvider.BlockInfosDb; var receiptsDb = dbProvider.ReceiptsDb; /* state & storage */ var stateProvider = new StateProvider(stateDb, codeDb, _logManager); var storageProvider = new StorageProvider(stateDb, stateProvider, _logManager); var ethereumSigner = new EthereumEcdsa(specProvider, _logManager); var transactionPool = new TxPool( NullTxStorage.Instance, Timestamper.Default, ethereumSigner, specProvider, new TxPoolConfig(), stateProvider, _logManager); var blockInfoRepository = new ChainLevelInfoRepository(blockInfosDb); var blockTree = new UnprocessedBlockTreeWrapper(new BlockTree(blocksDb, headersDb, blockInfosDb, blockInfoRepository, specProvider, transactionPool, _logManager)); var receiptStorage = new InMemoryReceiptStorage(); IBlockDataRecoveryStep recoveryStep = new TxSignaturesRecoveryStep(ethereumSigner, transactionPool, _logManager); /* blockchain processing */ IList <IAdditionalBlockProcessor> blockProcessors = new List <IAdditionalBlockProcessor>(); var blockhashProvider = new BlockhashProvider(blockTree, LimboLogs.Instance); var virtualMachine = new VirtualMachine(stateProvider, storageProvider, blockhashProvider, specProvider, _logManager); var processor = new TransactionProcessor(specProvider, stateProvider, storageProvider, virtualMachine, _logManager); ISealValidator sealValidator; if (specProvider.ChainId == RopstenSpecProvider.Instance.ChainId) { var difficultyCalculator = new DifficultyCalculator(specProvider); sealValidator = new EthashSealValidator(_logManager, difficultyCalculator, new Ethash(_logManager)); } else if (chainSpec.SealEngineType == SealEngineType.Clique) { var snapshotManager = new SnapshotManager(CliqueConfig.Default, blocksDb, blockTree, ethereumSigner, _logManager); sealValidator = new CliqueSealValidator(CliqueConfig.Default, snapshotManager, _logManager); rewardCalculator = NoBlockRewards.Instance; recoveryStep = new CompositeDataRecoveryStep(recoveryStep, new AuthorRecoveryStep(snapshotManager)); } else if (chainSpec.SealEngineType == SealEngineType.AuRa) { var abiEncoder = new AbiEncoder(); var validatorProcessor = new AuRaAdditionalBlockProcessorFactory(dbProvider.StateDb, stateProvider, abiEncoder, processor, blockTree, receiptStorage, _logManager) .CreateValidatorProcessor(chainSpec.AuRa.Validators); sealValidator = new AuRaSealValidator(chainSpec.AuRa, new AuRaStepCalculator(chainSpec.AuRa.StepDuration, new Timestamper()), validatorProcessor, ethereumSigner, _logManager); rewardCalculator = new AuRaRewardCalculator(chainSpec.AuRa, abiEncoder, processor); blockProcessors.Add(validatorProcessor); } else { throw new NotSupportedException(); } /* store & validation */ var headerValidator = new HeaderValidator(blockTree, sealValidator, specProvider, _logManager); var ommersValidator = new OmmersValidator(blockTree, headerValidator, _logManager); var transactionValidator = new TxValidator(chainSpec.ChainId); var blockValidator = new BlockValidator(transactionValidator, headerValidator, ommersValidator, specProvider, _logManager); /* blockchain processing */ var blockProcessor = new BlockProcessor(specProvider, blockValidator, rewardCalculator, processor, stateDb, codeDb, traceDb, stateProvider, storageProvider, transactionPool, receiptStorage, _logManager, blockProcessors); var blockchainProcessor = new BlockchainProcessor(blockTree, blockProcessor, recoveryStep, _logManager, true, false); if (chainSpec.SealEngineType == SealEngineType.AuRa) { stateProvider.CreateAccount(Address.Zero, UInt256.Zero); storageProvider.Commit(); stateProvider.Commit(Homestead.Instance); var finalizationManager = new AuRaBlockFinalizationManager(blockTree, blockInfoRepository, blockProcessor, blockProcessors.OfType <IAuRaValidator>().First(), _logManager); } foreach ((Address address, ChainSpecAllocation allocation) in chainSpec.Allocations) { stateProvider.CreateAccount(address, allocation.Balance); if (allocation.Code != null) { Keccak codeHash = stateProvider.UpdateCode(allocation.Code); stateProvider.UpdateCodeHash(address, codeHash, specProvider.GenesisSpec); } if (allocation.Constructor != null) { Transaction constructorTransaction = new Transaction(true) { SenderAddress = address, Init = allocation.Constructor, GasLimit = chainSpec.Genesis.GasLimit }; processor.Execute(constructorTransaction, chainSpec.Genesis.Header, NullTxTracer.Instance); } } _logger.Info($"Allocations configured, committing..."); stateProvider.Commit(specProvider.GenesisSpec); _logger.Info($"Finalizing genesis..."); chainSpec.Genesis.Header.StateRoot = stateProvider.StateRoot; chainSpec.Genesis.Header.Hash = BlockHeader.CalculateHash(chainSpec.Genesis.Header); if (chainSpec.Genesis.Hash != blockTree.Genesis.Hash) { throw new Exception("Unexpected genesis hash"); } _logger.Info($"Starting benchmark processor..."); /* start processing */ BigInteger totalGas = BigInteger.Zero; Stopwatch stopwatch = new Stopwatch(); Block currentHead; long maxMemory = 0; blockTree.NewHeadBlock += (sender, args) => { currentHead = args.Block; if (currentHead.Number == 0) { return; } maxMemory = Math.Max(maxMemory, GC.GetTotalMemory(false)); totalGas += currentHead.GasUsed; if ((BigInteger)args.Block.Number % 10000 == 9999) { stopwatch.Stop(); long ms = 1_000L * stopwatch.ElapsedTicks / Stopwatch.Frequency; BigInteger number = args.Block.Number + 1; _logger.Warn($"TOTAL after {number} (ms) : " + ms); _logger.Warn($"TOTAL after {number} blocks/s : {(decimal) currentHead.Number / (ms / 1000m),5}"); _logger.Warn($"TOTAL after {number} Mgas/s : {((decimal) totalGas / 1000000) / (ms / 1000m),5}"); _logger.Warn($"TOTAL after {number} max mem : {maxMemory}"); _logger.Warn($"TOTAL after {number} GC (0/1/2) : {GC.CollectionCount(0)}/{GC.CollectionCount(1)}/{GC.CollectionCount(2)}"); _logger.Warn($"Is server GC {number} : {System.Runtime.GCSettings.IsServerGC}"); _logger.Warn($"GC latency mode {number} : {System.Runtime.GCSettings.LatencyMode}"); _logger.Warn($"TOTAL after {number} blocks DB reads : {Store.Metrics.BlocksDbReads}"); _logger.Warn($"TOTAL after {number} blocks DB writes : {Store.Metrics.BlocksDbWrites}"); _logger.Warn($"TOTAL after {number} infos DB reads : {Store.Metrics.BlockInfosDbReads}"); _logger.Warn($"TOTAL after {number} infos DB writes : {Store.Metrics.BlockInfosDbWrites}"); _logger.Warn($"TOTAL after {number} state tree reads : {Store.Metrics.StateTreeReads}"); _logger.Warn($"TOTAL after {number} state tree writes : {Store.Metrics.StateTreeWrites}"); _logger.Warn($"TOTAL after {number} state DB reads : {Store.Metrics.StateDbReads}"); _logger.Warn($"TOTAL after {number} state DB writes : {Store.Metrics.StateDbWrites}"); _logger.Warn($"TOTAL after {number} storage tree reads : {Store.Metrics.StorageTreeReads}"); _logger.Warn($"TOTAL after {number} storage tree writes : {Store.Metrics.StorageTreeWrites}"); _logger.Warn($"TOTAL after {number} tree node hash : {Store.Metrics.TreeNodeHashCalculations}"); _logger.Warn($"TOTAL after {number} tree node RLP decode : {Store.Metrics.TreeNodeRlpDecodings}"); _logger.Warn($"TOTAL after {number} tree node RLP encode : {Store.Metrics.TreeNodeRlpEncodings}"); _logger.Warn($"TOTAL after {number} code DB reads : {Store.Metrics.CodeDbReads}"); _logger.Warn($"TOTAL after {number} code DB writes : {Store.Metrics.CodeDbWrites}"); _logger.Warn($"TOTAL after {number} receipts DB reads : {Store.Metrics.ReceiptsDbReads}"); _logger.Warn($"TOTAL after {number} receipts DB writes : {Store.Metrics.ReceiptsDbWrites}"); _logger.Warn($"TOTAL after {number} other DB reads : {Store.Metrics.OtherDbReads}"); _logger.Warn($"TOTAL after {number} other DB writes : {Store.Metrics.OtherDbWrites}"); _logger.Warn($"TOTAL after {number} EVM exceptions : {Evm.Metrics.EvmExceptions}"); _logger.Warn($"TOTAL after {number} SLOAD opcodes : {Evm.Metrics.SloadOpcode}"); _logger.Warn($"TOTAL after {number} SSTORE opcodes : {Evm.Metrics.SstoreOpcode}"); _logger.Warn($"TOTAL after {number} EXP opcodes : {Evm.Metrics.ModExpOpcode}"); _logger.Warn($"TOTAL after {number} BLOCKHASH opcodes : {Evm.Metrics.BlockhashOpcode}"); _logger.Warn($"TOTAL after {number} EVM calls : {Evm.Metrics.Calls}"); _logger.Warn($"TOTAL after {number} RIPEMD Precompiles : {Evm.Metrics.Ripemd160Precompile}"); _logger.Warn($"TOTAL after {number} SHA256 Precompiles : {Evm.Metrics.Sha256Precompile}"); // disk space stopwatch.Start(); } }; bool isStarted = false; TaskCompletionSource <object> completionSource = new TaskCompletionSource <object>(); blockTree.NewBestSuggestedBlock += (sender, args) => { if (!isStarted) { blockchainProcessor.Process(blockTree.FindBlock(blockTree.Genesis.Hash, BlockTreeLookupOptions.RequireCanonical), ProcessingOptions.None, NullBlockTracer.Instance); stopwatch.Start(); blockchainProcessor.Start(); isStarted = true; } if (args.Block.Number == BlocksToLoad) { completionSource.SetResult(null); } }; await Task.WhenAny(completionSource.Task, blockTree.LoadBlocksFromDb(CancellationToken.None, 0, 10000, BlocksToLoad)); await blockchainProcessor.StopAsync(true).ContinueWith( t => { if (t.IsFaulted) { _logger.Error("processing failed", t.Exception); _logger.Error("inner", t.Exception.InnerException); Console.ReadLine(); } _logger.Info("Block processing completed."); }); stopwatch.Stop(); Console.ReadLine(); }
private async Task InitBlockchain(InitParams initParams) { ChainSpec chainSpec = LoadChainSpec(initParams.ChainSpecPath); /* spec */ // TODO: rebuild to use chainspec ISpecProvider specProvider; if (chainSpec.ChainId == RopstenSpecProvider.Instance.ChainId) { specProvider = RopstenSpecProvider.Instance; } else if (chainSpec.ChainId == MainNetSpecProvider.Instance.ChainId) { specProvider = MainNetSpecProvider.Instance; } else { throw new NotSupportedException($"Not yet tested, not yet supported ChainId {chainSpec.ChainId}"); } var ethereumSigner = new EthereumSigner(specProvider, _logManager); var transactionStore = new TransactionStore(); var sealEngine = ConfigureSealEngine(transactionStore, ethereumSigner, initParams); /* sync */ var blocksDb = new DbOnTheRocks(Path.Combine(_dbBasePath, DbOnTheRocks.BlocksDbPath)); var blockInfosDb = new DbOnTheRocks(Path.Combine(_dbBasePath, DbOnTheRocks.BlockInfosDbPath)); var receiptsDb = new DbOnTheRocks(Path.Combine(_dbBasePath, DbOnTheRocks.ReceiptsDbPath)); /* blockchain */ var blockTree = new BlockTree(blocksDb, blockInfosDb, receiptsDb, specProvider, _logManager); var difficultyCalculator = new DifficultyCalculator(specProvider); /* validation */ var headerValidator = new HeaderValidator(difficultyCalculator, blockTree, sealEngine, specProvider, _logManager); var ommersValidator = new OmmersValidator(blockTree, headerValidator, _logManager); var txValidator = new TransactionValidator(new SignatureValidator(specProvider.ChainId)); var blockValidator = new BlockValidator(txValidator, headerValidator, ommersValidator, specProvider, _logManager); /* state */ var dbProvider = new RocksDbProvider(_dbBasePath, _logManager); var codeDb = new DbOnTheRocks(Path.Combine(_dbBasePath, DbOnTheRocks.CodeDbPath)); var stateDb = new DbOnTheRocks(Path.Combine(_dbBasePath, DbOnTheRocks.StateDbPath)); var stateTree = new StateTree(stateDb); var stateProvider = new StateProvider(stateTree, codeDb, _logManager); var storageProvider = new StorageProvider(dbProvider, stateProvider, _logManager); /* blockchain processing */ var blockhashProvider = new BlockhashProvider(blockTree); var virtualMachine = new VirtualMachine(stateProvider, storageProvider, blockhashProvider, _logManager); var transactionProcessor = new TransactionProcessor(specProvider, stateProvider, storageProvider, virtualMachine, _tracer, _logManager); var rewardCalculator = new RewardCalculator(specProvider); var blockProcessor = new BlockProcessor(specProvider, blockValidator, rewardCalculator, transactionProcessor, dbProvider, stateProvider, storageProvider, transactionStore, _logManager); _blockchainProcessor = new BlockchainProcessor(blockTree, sealEngine, transactionStore, difficultyCalculator, blockProcessor, ethereumSigner, _logManager); // create shared objects between discovery and peer manager _nodeFactory = new NodeFactory(); _nodeStatsProvider = new NodeStatsProvider(_configProvider); var jsonSerializer = new JsonSerializer(_logManager); var encrypter = new AesEncrypter(_configProvider, _logManager); _keyStore = new FileKeyStore(_configProvider, jsonSerializer, encrypter, _cryptoRandom, _logManager); //creating blockchain bridge BlockchainBridge = new BlockchainBridge(ethereumSigner, stateProvider, _keyStore, blockTree, stateDb, transactionStore); EthereumSigner = ethereumSigner; _blockchainProcessor.Start(); LoadGenesisBlock(chainSpec, string.IsNullOrWhiteSpace(initParams.GenesisHash) ? null : new Keccak(initParams.GenesisHash), blockTree, stateProvider, specProvider); await StartProcessing(blockTree, transactionStore, blockValidator, headerValidator, txValidator, initParams); }
private async Task InitBlockchain() { ChainSpec chainSpec = LoadChainSpec(_initConfig.ChainSpecPath); /* spec */ // TODO: rebuild to use chainspec ISpecProvider specProvider; if (chainSpec.ChainId == RopstenSpecProvider.Instance.ChainId) { specProvider = RopstenSpecProvider.Instance; } else if (chainSpec.ChainId == MainNetSpecProvider.Instance.ChainId) { specProvider = MainNetSpecProvider.Instance; } else { throw new NotSupportedException($"Not yet tested, not yet supported ChainId {chainSpec.ChainId}"); } var ethereumSigner = new EthereumSigner( specProvider, _logManager); var transactionStore = new TransactionStore(); var sealEngine = ConfigureSealEngine( transactionStore, ethereumSigner); /* sync */ IDbConfig dbConfig = _configProvider.GetConfig <IDbConfig>(); foreach (PropertyInfo propertyInfo in typeof(IDbConfig).GetProperties()) { _logger.Info($"DB {propertyInfo.Name}: {propertyInfo.GetValue(dbConfig)}"); } var blocksDb = new DbOnTheRocks( Path.Combine(_dbBasePath, DbOnTheRocks.BlocksDbPath), dbConfig); var blockInfosDb = new DbOnTheRocks( Path.Combine(_dbBasePath, DbOnTheRocks.BlockInfosDbPath), dbConfig); var receiptsDb = new DbOnTheRocks( Path.Combine(_dbBasePath, DbOnTheRocks.ReceiptsDbPath), dbConfig); /* blockchain */ _blockTree = new BlockTree( blocksDb, blockInfosDb, receiptsDb, specProvider, _logManager); var difficultyCalculator = new DifficultyCalculator( specProvider); /* validation */ var headerValidator = new HeaderValidator( difficultyCalculator, _blockTree, sealEngine, specProvider, _logManager); var ommersValidator = new OmmersValidator( _blockTree, headerValidator, _logManager); var txValidator = new TransactionValidator( new SignatureValidator(specProvider.ChainId)); var blockValidator = new BlockValidator( txValidator, headerValidator, ommersValidator, specProvider, _logManager); /* state */ var dbProvider = new RocksDbProvider( _dbBasePath, _logManager, dbConfig); var stateDb = dbProvider.GetOrCreateStateDb(); var stateTree = new StateTree(stateDb); var stateProvider = new StateProvider( stateTree, dbProvider.GetOrCreateCodeDb(), _logManager); var storageProvider = new StorageProvider( dbProvider, stateProvider, _logManager); /* blockchain processing */ var blockhashProvider = new BlockhashProvider( _blockTree); var virtualMachine = new VirtualMachine( stateProvider, storageProvider, blockhashProvider, _logManager); var transactionProcessor = new TransactionProcessor( specProvider, stateProvider, storageProvider, virtualMachine, _tracer, _logManager); var rewardCalculator = new RewardCalculator( specProvider); var blockProcessor = new BlockProcessor( specProvider, blockValidator, rewardCalculator, transactionProcessor, dbProvider, stateProvider, storageProvider, transactionStore, _logManager); _blockchainProcessor = new BlockchainProcessor( _blockTree, sealEngine, transactionStore, difficultyCalculator, blockProcessor, ethereumSigner, _logManager, _perfService); // create shared objects between discovery and peer manager _nodeFactory = new NodeFactory(); _nodeStatsProvider = new NodeStatsProvider(_configProvider.GetConfig <IStatsConfig>(), _nodeFactory, _logManager); var jsonSerializer = new JsonSerializer( _logManager); var encrypter = new AesEncrypter( _configProvider, _logManager); _keyStore = new FileKeyStore( _configProvider, jsonSerializer, encrypter, _cryptoRandom, _logManager); //creating blockchain bridge BlockchainBridge = new BlockchainBridge( ethereumSigner, stateProvider, _keyStore, _blockTree, stateDb, transactionStore); EthereumSigner = ethereumSigner; _blockchainProcessor.Start(); LoadGenesisBlock(chainSpec, string.IsNullOrWhiteSpace(_initConfig.GenesisHash) ? null : new Keccak(_initConfig.GenesisHash), _blockTree, stateProvider, specProvider); #pragma warning disable 4014 LoadBlocksFromDb(); #pragma warning restore 4014 await InitializeNetwork( transactionStore, blockValidator, headerValidator, txValidator); }