public void PruneElements(IEnumerable <KeyValuePair <UInt256, IEnumerable <int> > > blockTxIndices) { foreach (var keyPair in blockTxIndices) { var blockHash = keyPair.Key; var txIndices = keyPair.Value; ImmutableSortedDictionary <int, BlockTxNode> blockTxNodes; if (this.allBlockTxNodes.TryGetValue(blockHash, out blockTxNodes)) { var pruningCursor = new MemoryMerkleTreePruningCursor <BlockTxNode>(blockTxNodes.Values); foreach (var index in txIndices) { MerkleTree.PruneNode(pruningCursor, index); } var prunedBlockTxes = ImmutableSortedDictionary.CreateRange <int, BlockTxNode>( pruningCursor.ReadNodes().Select(blockTx => new KeyValuePair <int, BlockTxNode>(blockTx.Index, blockTx))); this.allBlockTxNodes[blockHash] = prunedBlockTxes; } } }
public void PruneElements(IEnumerable<KeyValuePair<UInt256, IEnumerable<int>>> blockTxIndices) { foreach (var keyPair in blockTxIndices) { var blockHash = keyPair.Key; var txIndices = keyPair.Value; ImmutableSortedDictionary<int, BlockTx> blockTxes; if (this.allBlockTxes.TryGetValue(blockHash, out blockTxes)) { var pruningCursor = new MemoryMerkleTreePruningCursor(blockTxes.Values); foreach (var index in txIndices) MerkleTree.PruneNode(pruningCursor, index); var prunedBlockTxes = ImmutableSortedDictionary.CreateRange<int, BlockTx>( pruningCursor.ReadNodes().Select(blockTx => new KeyValuePair<int, BlockTx>(blockTx.Index, blockTx))); this.allBlockTxes[blockHash] = prunedBlockTxes; } } }
public void TestPruneAllData() { var logger = LogManager.CreateNullLogger(); // create genesis block var genesisblock = CreateFakeBlock(1); var genesisHeader = new ChainedHeader(genesisblock.Header, height: 0, totalWork: genesisblock.Header.CalculateWork(), dateSeen: DateTimeOffset.Now); // create a block var txCount = 100; var block = CreateFakeBlock(txCount, genesisblock.Hash); var chainedHeader = ChainedHeader.CreateFromPrev(genesisHeader, block.Header, dateSeen: DateTimeOffset.Now); // create a long chain based off the block, to account for pruning buffer var fakeHeaders = new FakeHeaders(new[] { genesisHeader, chainedHeader }); var chain = new ChainBuilder(Enumerable.Concat(new[] { genesisHeader, chainedHeader }, Enumerable.Range(0, 2000).Select(x => fakeHeaders.NextChained()))).ToImmutable(); // mock core daemon to return the chain var coreDaemon = new Mock<ICoreDaemon>(); coreDaemon.Setup(x => x.CurrentChain).Returns(chain); // create memory storage with the block var storageManager = new MemoryStorageManager(); storageManager.BlockTxesStorage.TryAddBlockTransactions(block.Hash, block.BlockTxes); // initialize the pruning worker var workerConfig = new WorkerConfig(initialNotify: false, minIdleTime: TimeSpan.Zero, maxIdleTime: TimeSpan.MaxValue); using (var pruningWorker = new PruningWorker(workerConfig, coreDaemon.Object, storageManager, null)) // get a chain state cursor using (var handle = storageManager.OpenChainStateCursor()) { var chainStateCursor = handle.Item; // set the pruning worker to prune all data pruningWorker.Mode = PruningMode.TxIndex | PruningMode.BlockSpentIndex | PruningMode.BlockTxesPreserveMerkle; // wire event to wait for work var workFinishedEvent = new AutoResetEvent(false); pruningWorker.OnWorkFinished += () => workFinishedEvent.Set(); // wire event to track exceptions Exception workException = null; pruningWorker.OnWorkError += e => workException = e; // start the worker pruningWorker.Start(); // pick a random pruning order var random = new Random(); var pruneOrderSource = Enumerable.Range(0, txCount).ToList(); var pruneOrder = new List<int>(txCount); while (pruneOrderSource.Count > 0) { var randomIndex = random.Next(pruneOrderSource.Count); pruneOrder.Add(pruneOrderSource[randomIndex]); pruneOrderSource.RemoveAt(randomIndex); } // add an unspent tx for each transaction to storage var unspentTxes = new UnspentTx[block.Transactions.Length]; chainStateCursor.BeginTransaction(); for (var txIndex = 0; txIndex < block.Transactions.Length; txIndex++) { var tx = block.Transactions[txIndex]; var unspentTx = new UnspentTx(tx.Hash, blockIndex: 1, txIndex: txIndex, txVersion: tx.Version, isCoinbase: txIndex == 0, outputStates: new OutputStates(1, OutputState.Spent)); unspentTxes[txIndex] = unspentTx; chainStateCursor.TryAddUnspentTx(unspentTx); } chainStateCursor.CommitTransaction(); // create a memory pruning cursor to verify expected pruning results var pruneCursor = new MemoryMerkleTreePruningCursor<BlockTxNode>(block.BlockTxes.Select(x => (BlockTxNode)x)); // prune each transaction in random order var pruneHeight = 0; foreach (var pruneTxIndex in pruneOrder) { // create a spent tx to prune the transaction var pruneTx = block.Transactions[pruneTxIndex]; var spentTxes = BlockSpentTxes.CreateRange(new[] { unspentTxes[pruneTxIndex].ToSpentTx() }); // store the spent txes for the current pruning block pruneHeight++; chainStateCursor.BeginTransaction(); Assert.IsTrue(chainStateCursor.TryAddBlockSpentTxes(pruneHeight, spentTxes)); pruningWorker.PrunableHeight = pruneHeight; // verify unspent tx is present before pruning Assert.IsTrue(chainStateCursor.ContainsUnspentTx(pruneTx.Hash)); chainStateCursor.CommitTransaction(); // notify the pruning worker and wait pruningWorker.NotifyWork(); workFinishedEvent.WaitOne(); // verify unspent tx is removed after pruning chainStateCursor.BeginTransaction(); Assert.IsFalse(chainStateCursor.ContainsUnspentTx(pruneTx.Hash)); // verify unspent tx outputs are removed after pruning for (var outputIndex = 0; outputIndex < pruneTx.Outputs.Length; outputIndex++) Assert.IsFalse(chainStateCursor.ContainsUnspentTxOutput(new TxOutputKey(pruneTx.Hash, (uint)outputIndex))); // verify the spent txes were removed Assert.IsFalse(chainStateCursor.ContainsBlockSpentTxes(pruneHeight)); chainStateCursor.RollbackTransaction(); // prune to determine expected results MerkleTree.PruneNode(pruneCursor, pruneTxIndex); var expectedPrunedTxes = pruneCursor.ReadNodes().ToList(); // retrieve the actual transaction after pruning IEnumerator<BlockTxNode> actualPrunedTxNodes; Assert.IsTrue(storageManager.BlockTxesStorage.TryReadBlockTxNodes(block.Hash, out actualPrunedTxNodes)); // verify the actual pruned transactions match the expected results CollectionAssert.AreEqual(expectedPrunedTxes, actualPrunedTxNodes.UsingAsEnumerable().ToList()); } // verify all unspent txes were removed chainStateCursor.BeginTransaction(); Assert.AreEqual(0, chainStateCursor.ReadUnspentTransactions().Count()); chainStateCursor.CommitTransaction(); // verify final block with all transactions pruned IEnumerator<BlockTxNode> finalPrunedTxNodes; Assert.IsTrue(storageManager.BlockTxesStorage.TryReadBlockTxNodes(block.Hash, out finalPrunedTxNodes)); var finalPrunedTxesList = finalPrunedTxNodes.UsingAsEnumerable().ToList(); Assert.AreEqual(1, finalPrunedTxesList.Count); Assert.AreEqual(block.Header.MerkleRoot, finalPrunedTxesList.Single().Hash); // verify no work exceptions occurred Assert.IsNull(workException); } }
public void TestPruneMerkleTreeNodes() { var node1 = new MerkleTreeNode(index: 0, depth: 0, hash: (UInt256)1, pruned: false); var node2 = new MerkleTreeNode(index: 1, depth: 0, hash: (UInt256)2, pruned: false); var node3 = new MerkleTreeNode(index: 2, depth: 0, hash: (UInt256)3, pruned: false); var node4 = new MerkleTreeNode(index: 3, depth: 0, hash: (UInt256)4, pruned: false); var node5 = new MerkleTreeNode(index: 4, depth: 0, hash: (UInt256)5, pruned: false); var node6 = new MerkleTreeNode(index: 5, depth: 0, hash: (UInt256)6, pruned: false); var node7 = new MerkleTreeNode(index: 6, depth: 0, hash: (UInt256)7, pruned: false); var depth1Node1 = node1.AsPruned().PairWith(node2.AsPruned()); var depth1Node2 = node3.AsPruned().PairWith(node4.AsPruned()); var depth1Node3 = node5.AsPruned().PairWith(node6.AsPruned()); var depth1Node4 = node7.AsPruned().PairWithSelf(); var depth2Node1 = depth1Node1.PairWith(depth1Node2); var depth2Node2 = depth1Node3.PairWith(depth1Node4); var merkleRoot = depth2Node1.PairWith(depth2Node2); var nodes = new List<MerkleTreeNode> { node1, node2, node3, node4, node5, node6, node7 }; var cursor = new MemoryMerkleTreePruningCursor<MerkleTreeNode>(nodes); ////////////////////////////////////////////////// var expectedNodes1 = nodes; var actualNodes1 = cursor.ReadNodes().ToList(); CollectionAssert.AreEqual(expectedNodes1, actualNodes1); ////////////////////////////////////////////////// MerkleTree.PruneNode(cursor, 2); var expectedNodes2 = new List<MerkleTreeNode> { node1, node2, node3.AsPruned(), node4, node5, node6, node7 }; var actualNodes2 = cursor.ReadNodes().ToList(); CollectionAssert.AreEqual(expectedNodes2, actualNodes2); ////////////////////////////////////////////////// MerkleTree.PruneNode(cursor, 0); var expectedNodes3 = new List<MerkleTreeNode> { node1.AsPruned(), node2, node3.AsPruned(), node4, node5, node6, node7 }; var actualNodes3 = cursor.ReadNodes().ToList(); CollectionAssert.AreEqual(expectedNodes3, actualNodes3); ////////////////////////////////////////////////// MerkleTree.PruneNode(cursor, 1); var expectedNodes4 = new List<MerkleTreeNode> { depth1Node1, node3.AsPruned(), node4, node5, node6, node7 }; var actualNodes4 = cursor.ReadNodes().ToList(); CollectionAssert.AreEqual(expectedNodes4, actualNodes4); ////////////////////////////////////////////////// MerkleTree.PruneNode(cursor, 3); var expectedNodes5 = new List<MerkleTreeNode> { depth2Node1, node5, node6, node7 }; var actualNodes5 = cursor.ReadNodes().ToList(); CollectionAssert.AreEqual(expectedNodes5, actualNodes5); ////////////////////////////////////////////////// MerkleTree.PruneNode(cursor, 5); var expectedNodes6 = new List<MerkleTreeNode> { depth2Node1, node5, node6.AsPruned(), node7 }; var actualNodes6 = cursor.ReadNodes().ToList(); CollectionAssert.AreEqual(expectedNodes6, actualNodes6); ////////////////////////////////////////////////// MerkleTree.PruneNode(cursor, 6); var expectedNodes8 = new List<MerkleTreeNode> { depth2Node1, node5, node6.AsPruned(), depth1Node4 }; var actualNodes8 = cursor.ReadNodes().ToList(); CollectionAssert.AreEqual(expectedNodes8, actualNodes8); ////////////////////////////////////////////////// MerkleTree.PruneNode(cursor, 4); var expectedNodes9 = new List<MerkleTreeNode> { merkleRoot }; var actualNodes9 = cursor.ReadNodes().ToList(); CollectionAssert.AreEqual(expectedNodes9, actualNodes9); }