public CoreDaemon(ICoreRules rules, IStorageManager storageManager) { this.rules = rules; this.storageManager = storageManager; coreStorage = new CoreStorage(storageManager); // create chain state builder chainStateBuilder = new ChainStateBuilder(this.rules, coreStorage, this.storageManager); // create unconfirmed txes builder unconfirmedTxesBuilder = new UnconfirmedTxesBuilder(this, coreStorage, this.storageManager); // create workers targetChainWorker = new TargetChainWorker( new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMilliseconds(0), maxIdleTime: TimeSpan.FromSeconds(30)), ChainParams, coreStorage); chainStateWorker = new ChainStateWorker( new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMilliseconds(0), maxIdleTime: TimeSpan.FromSeconds(5)), targetChainWorker, chainStateBuilder, this.rules, coreStorage); unconfirmedTxesWorker = new UnconfirmedTxesWorker( new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMilliseconds(0), maxIdleTime: TimeSpan.FromSeconds(5)), chainStateWorker, unconfirmedTxesBuilder, coreStorage); pruningWorker = new PruningWorker( new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromSeconds(0), maxIdleTime: TimeSpan.FromMinutes(5)), this, this.storageManager, chainStateWorker); defragWorker = new DefragWorker( new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMinutes(5), maxIdleTime: TimeSpan.FromMinutes(5)), this.storageManager); gcWorker = new WorkerMethod("GC Worker", GcWorker, initialNotify: true, minIdleTime: TimeSpan.FromMinutes(5), maxIdleTime: TimeSpan.FromMinutes(5)); utxoScanWorker = new WorkerMethod("UTXO Scan Worker", UtxoScanWorker, initialNotify: true, minIdleTime: TimeSpan.FromSeconds(60), maxIdleTime: TimeSpan.FromSeconds(60)); statsWorker = new StatsWorker( new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMinutes(0), maxIdleTime: TimeSpan.MaxValue), this); // wire events chainStateWorker.BlockMissed += HandleBlockMissed; targetChainWorker.OnTargetChainChanged += HandleTargetChainChanged; chainStateWorker.OnChainStateChanged += HandleChainStateChanged; pruningWorker.OnWorkFinished += defragWorker.NotifyWork; unconfirmedTxesBuilder.UnconfirmedTxAdded += RaiseUnconfirmedTxAdded; unconfirmedTxesBuilder.TxesConfirmed += RaiseTxesConfirmed; unconfirmedTxesBuilder.TxesUnconfirmed += RaiseTxesUnconfirmed; }
public void TestPruneAllData() { // create genesis block var genesisblock = CreateFakeBlock(1); var genesisHeader = new ChainedHeader(genesisblock.Header, height: 0, totalWork: genesisblock.Header.CalculateWork().ToBigInteger(), dateSeen: DateTimeOffset.Now); // create a block var txCount = 100; var block = CreateFakeBlock(txCount, genesisblock.Hash); var chainedHeader = ChainedHeader.CreateFromPrev(genesisHeader, block.Header, dateSeen: DateTimeOffset.Now); // create a long chain based off the block, to account for pruning buffer var fakeHeaders = new FakeHeaders(new[] { genesisHeader, chainedHeader }); var chain = new ChainBuilder(Enumerable.Concat(new[] { genesisHeader, chainedHeader }, Enumerable.Range(0, 2000).Select(x => fakeHeaders.NextChained()))).ToImmutable(); // mock core daemon to return the chain var coreDaemon = new Mock <ICoreDaemon>(); coreDaemon.Setup(x => x.CurrentChain).Returns(chain); // create memory storage with the block var storageManager = new MemoryStorageManager(); storageManager.BlockTxesStorage.TryAddBlockTransactions(block.Hash, block.BlockTxes); // initialize the pruning worker var workerConfig = new WorkerConfig(initialNotify: false, minIdleTime: TimeSpan.Zero, maxIdleTime: TimeSpan.MaxValue); using (var pruningWorker = new PruningWorker(workerConfig, coreDaemon.Object, storageManager, null)) // get a chain state cursor using (var handle = storageManager.OpenChainStateCursor()) { var chainStateCursor = handle.Item; // set the pruning worker to prune all data pruningWorker.Mode = PruningMode.TxIndex | PruningMode.BlockSpentIndex | PruningMode.BlockTxesPreserveMerkle; // wire event to wait for work var workFinishedEvent = new AutoResetEvent(false); pruningWorker.OnWorkFinished += () => workFinishedEvent.Set(); // wire event to track exceptions Exception workException = null; pruningWorker.OnWorkError += e => workException = e; // start the worker pruningWorker.Start(); // pick a random pruning order var random = new Random(); var pruneOrderSource = Enumerable.Range(0, txCount).ToList(); var pruneOrder = new List <int>(txCount); while (pruneOrderSource.Count > 0) { var randomIndex = random.Next(pruneOrderSource.Count); pruneOrder.Add(pruneOrderSource[randomIndex]); pruneOrderSource.RemoveAt(randomIndex); } // add an unspent tx for each transaction to storage var unspentTxes = new UnspentTx[block.Transactions.Length]; chainStateCursor.BeginTransaction(); for (var txIndex = 0; txIndex < block.Transactions.Length; txIndex++) { var tx = block.Transactions[txIndex]; var unspentTx = new UnspentTx(tx.Hash, blockIndex: 1, txIndex: txIndex, txVersion: tx.Version, isCoinbase: txIndex == 0, outputStates: new OutputStates(1, OutputState.Spent)); unspentTxes[txIndex] = unspentTx; chainStateCursor.TryAddUnspentTx(unspentTx); } chainStateCursor.CommitTransaction(); // create a memory pruning cursor to verify expected pruning results var pruneCursor = new MemoryMerkleTreePruningCursor <BlockTxNode>(block.BlockTxes.Select(x => (BlockTxNode)x)); // prune each transaction in random order var pruneHeight = 0; foreach (var pruneTxIndex in pruneOrder) { // create a spent tx to prune the transaction var pruneTx = block.Transactions[pruneTxIndex]; var spentTxes = BlockSpentTxes.CreateRange(new[] { unspentTxes[pruneTxIndex].ToSpentTx() }); // store the spent txes for the current pruning block pruneHeight++; chainStateCursor.BeginTransaction(); Assert.IsTrue(chainStateCursor.TryAddBlockSpentTxes(pruneHeight, spentTxes)); pruningWorker.PrunableHeight = pruneHeight; // verify unspent tx is present before pruning Assert.IsTrue(chainStateCursor.ContainsUnspentTx(pruneTx.Hash)); chainStateCursor.CommitTransaction(); // notify the pruning worker and wait pruningWorker.NotifyWork(); workFinishedEvent.WaitOne(); // verify unspent tx is removed after pruning chainStateCursor.BeginTransaction(); Assert.IsFalse(chainStateCursor.ContainsUnspentTx(pruneTx.Hash)); // verify unspent tx outputs are removed after pruning for (var outputIndex = 0; outputIndex < pruneTx.Outputs.Length; outputIndex++) { Assert.IsFalse(chainStateCursor.ContainsUnspentTxOutput(new TxOutputKey(pruneTx.Hash, (uint)outputIndex))); } // verify the spent txes were removed Assert.IsFalse(chainStateCursor.ContainsBlockSpentTxes(pruneHeight)); chainStateCursor.RollbackTransaction(); // prune to determine expected results MerkleTree.PruneNode(pruneCursor, pruneTxIndex); var expectedPrunedTxes = pruneCursor.ReadNodes().ToList(); // retrieve the actual transaction after pruning IEnumerator <BlockTxNode> actualPrunedTxNodes; Assert.IsTrue(storageManager.BlockTxesStorage.TryReadBlockTxNodes(block.Hash, out actualPrunedTxNodes)); // verify the actual pruned transactions match the expected results CollectionAssert.AreEqual(expectedPrunedTxes, actualPrunedTxNodes.UsingAsEnumerable().ToList()); } // verify all unspent txes were removed chainStateCursor.BeginTransaction(); Assert.AreEqual(0, chainStateCursor.ReadUnspentTransactions().Count()); chainStateCursor.CommitTransaction(); // verify final block with all transactions pruned IEnumerator <BlockTxNode> finalPrunedTxNodes; Assert.IsTrue(storageManager.BlockTxesStorage.TryReadBlockTxNodes(block.Hash, out finalPrunedTxNodes)); var finalPrunedTxesList = finalPrunedTxNodes.UsingAsEnumerable().ToList(); Assert.AreEqual(1, finalPrunedTxesList.Count); Assert.AreEqual(block.Header.MerkleRoot, finalPrunedTxesList.Single().Hash); // verify no work exceptions occurred Assert.IsNull(workException); } }
public CoreDaemon(Logger logger, IKernel kernel, IBlockchainRules rules, IStorageManager storageManager) { this.logger = logger; this.shutdownToken = new CancellationTokenSource(); this.kernel = kernel; this.rules = rules; this.storageManager = storageManager; this.coreStorage = new CoreStorage(storageManager, logger); // write genesis block out to storage this.coreStorage.AddGenesisBlock(this.rules.GenesisChainedHeader); this.coreStorage.TryAddBlock(this.rules.GenesisBlock); // create chain state builder this.chainStateBuilder = new ChainStateBuilder(this.logger, this.rules, this.coreStorage); // add genesis block to chain state, if needed if (this.chainStateBuilder.Chain.Height < 0) { this.chainStateBuilder.AddBlock(this.rules.GenesisChainedHeader, this.rules.GenesisBlock.Transactions); } // create workers this.targetChainWorker = new TargetChainWorker( new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMilliseconds(50), maxIdleTime: TimeSpan.FromSeconds(30)), this.logger, this.rules, this.coreStorage); this.chainStateWorker = new ChainStateWorker( new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMilliseconds(50), maxIdleTime: TimeSpan.FromSeconds(5)), this.targetChainWorker, this.chainStateBuilder, this.logger, this.rules, this.coreStorage); this.pruningWorker = new PruningWorker( new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromSeconds(60), maxIdleTime: TimeSpan.FromMinutes(15)), this.coreStorage, this.chainStateWorker, this.chainStateBuilder, this.logger, this.rules); this.defragWorker = new DefragWorker( new WorkerConfig(initialNotify: true, minIdleTime: TimeSpan.FromMinutes(5), maxIdleTime: TimeSpan.FromMinutes(5)), this.coreStorage, this.logger); // notify defrag worker after pruning this.pruningWorker.OnWorkFinished += this.defragWorker.NotifyWork; this.chainStateWorker.BlockMissed += HandleBlockMissed; this.targetChainWorker.OnTargetChainChanged += () => { var handler = this.OnTargetChainChanged; if (handler != null) { handler(this, EventArgs.Empty); } }; this.chainStateWorker.OnChainStateChanged += () => { this.pruningWorker.NotifyWork(); this.utxoScanWorker.NotifyWork(); var handler = this.OnChainStateChanged; if (handler != null) { handler(this, EventArgs.Empty); } }; this.gcWorker = new WorkerMethod("GC Worker", _ => { this.logger.Info( string.Join("\n", new string('-', 80), "GC Memory: {0,10:#,##0.00} MB", "Process Memory: {1,10:#,##0.00} MB", new string('-', 80) ) .Format2 ( /*0*/ (float)GC.GetTotalMemory(false) / 1.MILLION(), /*1*/ (float)Process.GetCurrentProcess().PrivateMemorySize64 / 1.MILLION() )); }, initialNotify: true, minIdleTime: TimeSpan.FromSeconds(30), maxIdleTime: TimeSpan.FromSeconds(30), logger: this.logger); this.utxoScanWorker = new WorkerMethod("UTXO Scan Worker", _ => { // time taking chain state snapshots var stopwatch = Stopwatch.StartNew(); int chainStateHeight; using (var chainState = this.GetChainState()) { chainStateHeight = chainState.Chain.Height; } stopwatch.Stop(); this.logger.Info("GetChainState at {0:#,##0}: {1:#,##0.00}s".Format2(chainStateHeight, stopwatch.Elapsed.TotalSeconds)); // time enumerating chain state snapshots stopwatch = Stopwatch.StartNew(); using (var chainState = this.GetChainState()) { chainStateHeight = chainState.Chain.Height; chainState.ReadUnspentTransactions().Count(); } stopwatch.Stop(); this.logger.Info("Enumerate chain state at {0:#,##0}: {1:#,##0.00}s".Format2(chainStateHeight, stopwatch.Elapsed.TotalSeconds)); //using (var chainStateLocal = this.GetChainState()) //{ // new MethodTimer(this.logger).Time("UTXO Commitment: {0:#,##0}".Format2(chainStateLocal.UnspentTxCount), () => // { // using (var utxoStream = new UtxoStream(this.logger, chainStateLocal.ReadUnspentTransactions())) // { // var sha256 = new SHA256Managed(); // var utxoHash = sha256.ComputeHash(utxoStream); // this.logger.Info("UXO Commitment Hash: {0}".Format2(utxoHash.ToHexNumberString())); // } // }); // //new MethodTimer().Time("Full UTXO Scan: {0:#,##0}".Format2(chainStateLocal.Utxo.TransactionCount), () => // //{ // // var sha256 = new SHA256Managed(); // // foreach (var output in chainStateLocal.Utxo.GetUnspentTransactions()) // // { // // } // //}); //} }, initialNotify: true, minIdleTime: TimeSpan.FromSeconds(60), maxIdleTime: TimeSpan.FromSeconds(60), logger: this.logger); }