public static async Task <PruningTestBlockchain> Create(IPruningConfig pruningConfig = null)
{
PruningTestBlockchain chain = new() { PruningConfig = pruningConfig ?? new PruningConfig() };
await chain.Build();
return(chain);
}
public async Task Execute(CancellationToken _)
{
ILogger logger = _api.LogManager.GetClassLogger();
/* sync */
IDbConfig dbConfig = _api.Config <IDbConfig>();
ISyncConfig syncConfig = _api.Config <ISyncConfig>();
IInitConfig initConfig = _api.Config <IInitConfig>();
IPruningConfig pruningConfig = _api.Config <IPruningConfig>();
foreach (PropertyInfo propertyInfo in typeof(IDbConfig).GetProperties())
{
if (logger.IsDebug)
{
logger.Debug($"DB {propertyInfo.Name}: {propertyInfo.GetValue(dbConfig)}");
}
}
try
{
bool useReceiptsDb = initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync;
InitDbApi(initConfig, dbConfig, initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync);
StandardDbInitializer dbInitializer = new(_api.DbProvider, _api.RocksDbFactory, _api.MemDbFactory, _api.FileSystem, pruningConfig.Mode.IsFull());
await dbInitializer.InitStandardDbsAsync(useReceiptsDb);
}
catch (TypeInitializationException e)
{
if (logger.IsError)
{
logger.Error("RocksDb was not found, please make sure it is installed on your machine. \n On macOs : 'brew install rocksdb'", e);
}
}
}
public FullTestPruner(
IFullPruningDb pruningDb,
IPruningTrigger pruningTrigger,
IPruningConfig pruningConfig,
IBlockTree blockTree,
IStateReader stateReader,
ILogManager logManager)
: base(pruningDb, pruningTrigger, pruningConfig, blockTree, stateReader, logManager)
{
}
public FullPruner(
IFullPruningDb fullPruningDb,
IPruningTrigger pruningTrigger,
IPruningConfig pruningConfig,
IBlockTree blockTree,
IStateReader stateReader,
ILogManager logManager)
{
_fullPruningDb = fullPruningDb;
_pruningTrigger = pruningTrigger;
_pruningConfig = pruningConfig;
_blockTree = blockTree;
_stateReader = stateReader;
_logManager = logManager;
_pruningTrigger.Prune += OnPrune;
_logger = _logManager.GetClassLogger();
_minimumPruningDelay = TimeSpan.FromHours(_pruningConfig.FullPruningMinimumDelayHours);
}
private static void InitializeFullPruning(
IPruningConfig pruningConfig,
IInitConfig initConfig,
INethermindApi api,
IStateReader stateReader)
{
IPruningTrigger?CreateAutomaticTrigger(string dbPath)
{
long threshold = pruningConfig.FullPruningThresholdMb.MB();
switch (pruningConfig.FullPruningTrigger)
{
case FullPruningTrigger.StateDbSize:
return(new PathSizePruningTrigger(dbPath, threshold, api.TimerFactory, api.FileSystem));
case FullPruningTrigger.VolumeFreeSpace:
return(new DiskFreeSpacePruningTrigger(dbPath, threshold, api.TimerFactory, api.FileSystem));
default:
return(null);
}
}
if (pruningConfig.Mode.IsFull())
{
IDb stateDb = api.DbProvider !.StateDb;
if (stateDb is IFullPruningDb fullPruningDb)
{
IPruningTrigger?pruningTrigger = CreateAutomaticTrigger(fullPruningDb.GetPath(initConfig.BaseDbPath));
if (pruningTrigger is not null)
{
api.PruningTrigger.Add(pruningTrigger);
}
FullPruner pruner = new(fullPruningDb, api.PruningTrigger, pruningConfig, api.BlockTree !, stateReader, api.LogManager);
api.DisposeStack.Push(pruner);
}
}
}
public FullPruner(
IFullPruningDb fullPruningDb,
IPruningTrigger pruningTrigger,
IPruningConfig pruningConfig,
IBlockTree blockTree,
IStateReader stateReader,
ILogManager logManager)
{
_fullPruningDb = fullPruningDb;
_pruningTrigger = pruningTrigger;
_pruningConfig = pruningConfig;
_blockTree = blockTree;
_stateReader = stateReader;
_logManager = logManager;
_pruningTrigger.Prune += OnPrune;
_logger = _logManager.GetClassLogger();
_minimumPruningDelay = TimeSpan.FromHours(_pruningConfig.FullPruningMinimumDelayHours);
if (_pruningConfig.FullPruningCompletionBehavior != FullPruningCompletionBehavior.None)
{
_fullPruningDb.PruningFinished += HandlePruningFinished;
}
}
private Task InitBlockchain()
{
BlockTraceDumper.Converters.AddRange(DebugModuleFactory.Converters);
BlockTraceDumper.Converters.AddRange(TraceModuleFactory.Converters);
var(getApi, setApi) = _api.ForBlockchain;
if (getApi.ChainSpec == null)
{
throw new StepDependencyException(nameof(getApi.ChainSpec));
}
if (getApi.DbProvider == null)
{
throw new StepDependencyException(nameof(getApi.DbProvider));
}
if (getApi.SpecProvider == null)
{
throw new StepDependencyException(nameof(getApi.SpecProvider));
}
_logger = getApi.LogManager.GetClassLogger();
IInitConfig initConfig = getApi.Config <IInitConfig>();
ISyncConfig syncConfig = getApi.Config <ISyncConfig>();
IPruningConfig pruningConfig = getApi.Config <IPruningConfig>();
if (syncConfig.DownloadReceiptsInFastSync && !syncConfig.DownloadBodiesInFastSync)
{
_logger.Warn($"{nameof(syncConfig.DownloadReceiptsInFastSync)} is selected but {nameof(syncConfig.DownloadBodiesInFastSync)} - enabling bodies to support receipts download.");
syncConfig.DownloadBodiesInFastSync = true;
}
Account.AccountStartNonce = getApi.ChainSpec.Parameters.AccountStartNonce;
IWitnessCollector witnessCollector = setApi.WitnessCollector = syncConfig.WitnessProtocolEnabled
? new WitnessCollector(getApi.DbProvider.WitnessDb, _api.LogManager)
.WithPruning(getApi.BlockTree !, getApi.LogManager)
: NullWitnessCollector.Instance;
IKeyValueStoreWithBatching cachedStateDb = getApi.DbProvider.StateDb
.Cached(Trie.MemoryAllowance.TrieNodeCacheCount);
setApi.MainStateDbWithCache = cachedStateDb;
IKeyValueStore codeDb = getApi.DbProvider.CodeDb
.WitnessedBy(witnessCollector);
TrieStore trieStore;
if (pruningConfig.Enabled)
{
setApi.TrieStore = trieStore = new TrieStore(
setApi.MainStateDbWithCache.WitnessedBy(witnessCollector),
Prune.WhenCacheReaches(pruningConfig.CacheMb.MB()), // TODO: memory hint should define this
Persist.IfBlockOlderThan(pruningConfig.PersistenceInterval), // TODO: this should be based on time
getApi.LogManager);
}
else
{
setApi.TrieStore = trieStore = new TrieStore(
setApi.MainStateDbWithCache.WitnessedBy(witnessCollector),
No.Pruning,
Persist.EveryBlock,
getApi.LogManager);
}
getApi.DisposeStack.Push(trieStore);
trieStore.ReorgBoundaryReached += ReorgBoundaryReached;
ITrieStore readOnlyTrieStore = setApi.ReadOnlyTrieStore = trieStore.AsReadOnly(cachedStateDb);
IStateProvider stateProvider = setApi.StateProvider = new StateProvider(
trieStore,
codeDb,
getApi.LogManager);
ReadOnlyDbProvider readOnly = new(getApi.DbProvider, false);
PersistentTxStorage txStorage = new(getApi.DbProvider.PendingTxsDb);
IStateReader stateReader = setApi.StateReader = new StateReader(readOnlyTrieStore, readOnly.GetDb <IDb>(DbNames.Code), getApi.LogManager);
setApi.TransactionComparerProvider =
new TransactionComparerProvider(getApi.SpecProvider !, getApi.BlockTree.AsReadOnly());
setApi.ChainHeadStateProvider = new ChainHeadReadOnlyStateProvider(getApi.BlockTree, stateReader);
Account.AccountStartNonce = getApi.ChainSpec.Parameters.AccountStartNonce;
stateProvider.StateRoot = getApi.BlockTree !.Head?.StateRoot ?? Keccak.EmptyTreeHash;
if (_api.Config <IInitConfig>().DiagnosticMode == DiagnosticMode.VerifyTrie)
{
_logger.Info("Collecting trie stats and verifying that no nodes are missing...");
TrieStats stats = stateProvider.CollectStats(getApi.DbProvider.CodeDb, _api.LogManager);
_logger.Info($"Starting from {getApi.BlockTree.Head?.Number} {getApi.BlockTree.Head?.StateRoot}{Environment.NewLine}" + stats);
}
// Init state if we need system calls before actual processing starts
if (getApi.BlockTree !.Head?.StateRoot != null)
{
stateProvider.StateRoot = getApi.BlockTree.Head.StateRoot;
}
var txValidator = setApi.TxValidator = new TxValidator(getApi.SpecProvider.ChainId);
ITxPool txPool = _api.TxPool = CreateTxPool(txStorage);
OnChainTxWatcher onChainTxWatcher = new(getApi.BlockTree, txPool, getApi.SpecProvider, _api.LogManager);
getApi.DisposeStack.Push(onChainTxWatcher);
ReceiptCanonicalityMonitor receiptCanonicalityMonitor = new(getApi.BlockTree, getApi.ReceiptStorage, _api.LogManager);
getApi.DisposeStack.Push(receiptCanonicalityMonitor);
_api.BlockPreprocessor.AddFirst(
new RecoverSignatures(getApi.EthereumEcdsa, txPool, getApi.SpecProvider, getApi.LogManager));
IStorageProvider storageProvider = setApi.StorageProvider = new StorageProvider(
trieStore,
stateProvider,
getApi.LogManager);
// blockchain processing
BlockhashProvider blockhashProvider = new BlockhashProvider(
getApi.BlockTree, getApi.LogManager);
VirtualMachine virtualMachine = new VirtualMachine(
stateProvider,
storageProvider,
blockhashProvider,
getApi.SpecProvider,
getApi.LogManager);
_api.TransactionProcessor = new TransactionProcessor(
getApi.SpecProvider,
stateProvider,
storageProvider,
virtualMachine,
getApi.LogManager);
InitSealEngine();
if (_api.SealValidator == null)
{
throw new StepDependencyException(nameof(_api.SealValidator));
}
/* validation */
var headerValidator = setApi.HeaderValidator = CreateHeaderValidator();
OmmersValidator ommersValidator = new(
getApi.BlockTree,
headerValidator,
getApi.LogManager);
var blockValidator = setApi.BlockValidator = new BlockValidator(
txValidator,
headerValidator,
ommersValidator,
getApi.SpecProvider,
getApi.LogManager);
setApi.TxPoolInfoProvider = new TxPoolInfoProvider(stateReader, txPool);
var mainBlockProcessor = setApi.MainBlockProcessor = CreateBlockProcessor();
BlockchainProcessor blockchainProcessor = new(
getApi.BlockTree,
mainBlockProcessor,
_api.BlockPreprocessor,
getApi.LogManager,
new BlockchainProcessor.Options
{
AutoProcess = !syncConfig.BeamSync,
StoreReceiptsByDefault = initConfig.StoreReceipts,
});
setApi.BlockProcessingQueue = blockchainProcessor;
setApi.BlockchainProcessor = blockchainProcessor;
if (syncConfig.BeamSync)
{
BeamBlockchainProcessor beamBlockchainProcessor = new(
new ReadOnlyDbProvider(_api.DbProvider, false),
getApi.BlockTree,
getApi.SpecProvider,
getApi.LogManager,
blockValidator,
_api.BlockPreprocessor,
_api.RewardCalculatorSource !, // TODO: does it work with AuRa?
blockchainProcessor,
getApi.SyncModeSelector !);
_api.DisposeStack.Push(beamBlockchainProcessor);
}
// TODO: can take the tx sender from plugin here maybe
ITxSigner txSigner = new WalletTxSigner(getApi.Wallet, getApi.SpecProvider.ChainId);
TxSealer standardSealer = new(txSigner, getApi.Timestamper);
NonceReservingTxSealer nonceReservingTxSealer =
new(txSigner, getApi.Timestamper, txPool);
setApi.TxSender = new TxPoolSender(txPool, nonceReservingTxSealer, standardSealer);
// TODO: possibly hide it (but need to confirm that NDM does not really need it)
var filterStore = setApi.FilterStore = new FilterStore();
setApi.FilterManager = new FilterManager(filterStore, mainBlockProcessor, txPool, getApi.LogManager);
setApi.HealthHintService = CreateHealthHintService();
return(Task.CompletedTask);
}
private Task InitBlockchain()
{
InitBlockTraceDumper();
(IApiWithStores getApi, IApiWithBlockchain setApi) = _api.ForBlockchain;
if (getApi.ChainSpec == null)
{
throw new StepDependencyException(nameof(getApi.ChainSpec));
}
if (getApi.DbProvider == null)
{
throw new StepDependencyException(nameof(getApi.DbProvider));
}
if (getApi.SpecProvider == null)
{
throw new StepDependencyException(nameof(getApi.SpecProvider));
}
if (getApi.BlockTree == null)
{
throw new StepDependencyException(nameof(getApi.BlockTree));
}
_logger = getApi.LogManager.GetClassLogger();
IInitConfig initConfig = getApi.Config <IInitConfig>();
ISyncConfig syncConfig = getApi.Config <ISyncConfig>();
IPruningConfig pruningConfig = getApi.Config <IPruningConfig>();
IMiningConfig miningConfig = getApi.Config <IMiningConfig>();
if (syncConfig.DownloadReceiptsInFastSync && !syncConfig.DownloadBodiesInFastSync)
{
_logger.Warn($"{nameof(syncConfig.DownloadReceiptsInFastSync)} is selected but {nameof(syncConfig.DownloadBodiesInFastSync)} - enabling bodies to support receipts download.");
syncConfig.DownloadBodiesInFastSync = true;
}
Account.AccountStartNonce = getApi.ChainSpec.Parameters.AccountStartNonce;
IWitnessCollector witnessCollector;
if (syncConfig.WitnessProtocolEnabled)
{
WitnessCollector witnessCollectorImpl = new(getApi.DbProvider.WitnessDb, _api.LogManager);
witnessCollector = setApi.WitnessCollector = witnessCollectorImpl;
setApi.WitnessRepository = witnessCollectorImpl.WithPruning(getApi.BlockTree !, getApi.LogManager);
}
else
{
witnessCollector = setApi.WitnessCollector = NullWitnessCollector.Instance;
setApi.WitnessRepository = NullWitnessCollector.Instance;
}
CachingStore cachedStateDb = getApi.DbProvider.StateDb
.Cached(Trie.MemoryAllowance.TrieNodeCacheCount);
setApi.MainStateDbWithCache = cachedStateDb;
IKeyValueStore codeDb = getApi.DbProvider.CodeDb
.WitnessedBy(witnessCollector);
TrieStore trieStore;
IKeyValueStoreWithBatching stateWitnessedBy = setApi.MainStateDbWithCache.WitnessedBy(witnessCollector);
if (pruningConfig.Mode.IsMemory())
{
IPersistenceStrategy persistenceStrategy = Persist.IfBlockOlderThan(pruningConfig.PersistenceInterval); // TODO: this should be based on time
if (pruningConfig.Mode.IsFull())
{
PruningTriggerPersistenceStrategy triggerPersistenceStrategy = new((IFullPruningDb)getApi.DbProvider !.StateDb, getApi.BlockTree !, getApi.LogManager);
getApi.DisposeStack.Push(triggerPersistenceStrategy);
persistenceStrategy = persistenceStrategy.Or(triggerPersistenceStrategy);
}
setApi.TrieStore = trieStore = new TrieStore(
stateWitnessedBy,
Prune.WhenCacheReaches(pruningConfig.CacheMb.MB()), // TODO: memory hint should define this
persistenceStrategy,
getApi.LogManager);
if (pruningConfig.Mode.IsFull())
{
IFullPruningDb fullPruningDb = (IFullPruningDb)getApi.DbProvider !.StateDb;
fullPruningDb.PruningStarted += (_, args) =>
{
cachedStateDb.PersistCache(args.Context);
trieStore.PersistCache(args.Context, args.Context.CancellationTokenSource.Token);
};
}
}
else
{
setApi.TrieStore = trieStore = new TrieStore(
stateWitnessedBy,
No.Pruning,
Persist.EveryBlock,
getApi.LogManager);
}
TrieStoreBoundaryWatcher trieStoreBoundaryWatcher = new(trieStore, _api.BlockTree !, _api.LogManager);
getApi.DisposeStack.Push(trieStoreBoundaryWatcher);
getApi.DisposeStack.Push(trieStore);
ITrieStore readOnlyTrieStore = setApi.ReadOnlyTrieStore = trieStore.AsReadOnly(cachedStateDb);
IStateProvider stateProvider = setApi.StateProvider = new StateProvider(
trieStore,
codeDb,
getApi.LogManager);
ReadOnlyDbProvider readOnly = new(getApi.DbProvider, false);
IStateReader stateReader = setApi.StateReader = new StateReader(readOnlyTrieStore, readOnly.GetDb <IDb>(DbNames.Code), getApi.LogManager);
setApi.TransactionComparerProvider = new TransactionComparerProvider(getApi.SpecProvider !, getApi.BlockTree.AsReadOnly());
setApi.ChainHeadStateProvider = new ChainHeadReadOnlyStateProvider(getApi.BlockTree, stateReader);
Account.AccountStartNonce = getApi.ChainSpec.Parameters.AccountStartNonce;
stateProvider.StateRoot = getApi.BlockTree !.Head?.StateRoot ?? Keccak.EmptyTreeHash;
if (_api.Config <IInitConfig>().DiagnosticMode == DiagnosticMode.VerifyTrie)
{
Task.Run(() =>
{
try
{
_logger !.Info("Collecting trie stats and verifying that no nodes are missing...");
TrieStats stats = stateProvider.CollectStats(getApi.DbProvider.CodeDb, _api.LogManager);
_logger.Info($"Starting from {getApi.BlockTree.Head?.Number} {getApi.BlockTree.Head?.StateRoot}{Environment.NewLine}" + stats);
}
catch (Exception ex)
{
_logger !.Error(ex.ToString());
}
});
}
// Init state if we need system calls before actual processing starts
if (getApi.BlockTree !.Head?.StateRoot != null)
{
stateProvider.StateRoot = getApi.BlockTree.Head.StateRoot;
}
TxValidator txValidator = setApi.TxValidator = new TxValidator(getApi.SpecProvider.ChainId);
ITxPool txPool = _api.TxPool = CreateTxPool();
ReceiptCanonicalityMonitor receiptCanonicalityMonitor = new(getApi.BlockTree, getApi.ReceiptStorage, _api.LogManager);
getApi.DisposeStack.Push(receiptCanonicalityMonitor);
_api.ReceiptMonitor = receiptCanonicalityMonitor;
_api.BlockPreprocessor.AddFirst(
new RecoverSignatures(getApi.EthereumEcdsa, txPool, getApi.SpecProvider, getApi.LogManager));
IStorageProvider storageProvider = setApi.StorageProvider = new StorageProvider(
trieStore,
stateProvider,
getApi.LogManager);
// blockchain processing
BlockhashProvider blockhashProvider = new (
getApi.BlockTree, getApi.LogManager);
VirtualMachine virtualMachine = new (
blockhashProvider,
getApi.SpecProvider,
getApi.LogManager);
WorldState worldState = new (stateProvider, storageProvider);
_api.TransactionProcessor = new TransactionProcessor(
getApi.SpecProvider,
worldState,
virtualMachine,
getApi.LogManager);
InitSealEngine();
if (_api.SealValidator == null)
{
throw new StepDependencyException(nameof(_api.SealValidator));
}
setApi.HeaderValidator = CreateHeaderValidator();
IHeaderValidator?headerValidator = setApi.HeaderValidator;
IUnclesValidator unclesValidator = setApi.UnclesValidator = new UnclesValidator(
getApi.BlockTree,
headerValidator,
getApi.LogManager);
setApi.BlockValidator = new BlockValidator(
txValidator,
headerValidator,
unclesValidator,
getApi.SpecProvider,
getApi.LogManager);
IChainHeadInfoProvider chainHeadInfoProvider =
new ChainHeadInfoProvider(getApi.SpecProvider, getApi.BlockTree, stateReader);
setApi.TxPoolInfoProvider = new TxPoolInfoProvider(chainHeadInfoProvider.AccountStateProvider, txPool);
setApi.GasPriceOracle = new GasPriceOracle(getApi.BlockTree, getApi.SpecProvider, _api.LogManager, miningConfig.MinGasPrice);
IBlockProcessor mainBlockProcessor = setApi.MainBlockProcessor = CreateBlockProcessor();
BlockchainProcessor blockchainProcessor = new(
getApi.BlockTree,
mainBlockProcessor,
_api.BlockPreprocessor,
stateReader,
getApi.LogManager,
new BlockchainProcessor.Options
{
StoreReceiptsByDefault = initConfig.StoreReceipts,
DumpOptions = initConfig.AutoDump
});
setApi.BlockProcessingQueue = blockchainProcessor;
setApi.BlockchainProcessor = blockchainProcessor;
setApi.EthSyncingInfo = new EthSyncingInfo(getApi.BlockTree);
// TODO: can take the tx sender from plugin here maybe
ITxSigner txSigner = new WalletTxSigner(getApi.Wallet, getApi.SpecProvider.ChainId);
TxSealer standardSealer = new(txSigner, getApi.Timestamper);
NonceReservingTxSealer nonceReservingTxSealer =
new(txSigner, getApi.Timestamper, txPool);
setApi.TxSender = new TxPoolSender(txPool, nonceReservingTxSealer, standardSealer);
// TODO: possibly hide it (but need to confirm that NDM does not really need it)
IFilterStore filterStore = setApi.FilterStore = new FilterStore();
setApi.FilterManager = new FilterManager(filterStore, mainBlockProcessor, txPool, getApi.LogManager);
setApi.HealthHintService = CreateHealthHintService();
setApi.BlockProductionPolicy = new BlockProductionPolicy(miningConfig);
InitializeFullPruning(pruningConfig, initConfig, _api, stateReader);
return(Task.CompletedTask);
}
