public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes) { CheckWriteTransaction(); try { using (SetSessionContext()) using (var jetUpdate = this.jetSession.BeginUpdate(this.spentTxTableId, JET_prep.Insert)) { byte[] spentTxesBytes; using (var stream = new MemoryStream()) using (var writer = new BinaryWriter(stream)) { writer.WriteList <SpentTx>(spentTxes, spentTx => DataEncoder.EncodeSpentTx(writer, spentTx)); spentTxesBytes = stream.ToArray(); } Api.SetColumns(this.jetSession, this.spentTxTableId, new Int32ColumnValue { Columnid = this.spentSpentBlockIndexColumnId, Value = blockIndex }, new BytesColumnValue { Columnid = this.spentDataColumnId, Value = spentTxesBytes }); jetUpdate.Save(); } return(true); } catch (EsentKeyDuplicateException) { return(false); } }
public static byte[] EncodeBlockSpentTxes(BlockSpentTxes blockSpentTxes) { using (var stream = new MemoryStream()) using (var writer = new BinaryWriter(stream)) { EncodeBlockSpentTxes(writer, blockSpentTxes); return(stream.ToArray()); } }
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes) { CheckWriteTransaction(); try { blockSpentTxes.Modify(x => x.Add(blockIndex, spentTxes)); return(true); } catch (ArgumentException) { return(false); } }
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes) { CheckTransaction(); using (SetSessionContext()) { Api.JetSetCurrentIndex(this.jetSession, this.spentTxTableId, "IX_SpentBlockIndex"); Api.MakeKey(this.jetSession, this.spentTxTableId, blockIndex, MakeKeyGrbit.NewKey); if (Api.TrySeek(this.jetSession, this.spentTxTableId, SeekGrbit.SeekEQ)) { var spentTxesBytes = Api.RetrieveColumn(this.jetSession, this.spentTxTableId, this.spentDataColumnId); spentTxes = DataDecoder.DecodeBlockSpentTxes(spentTxesBytes); return(true); } else { spentTxes = null; return(false); } } }
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes) { CheckWriteTransaction(); var key = DbEncoder.EncodeInt32(blockIndex); if (!this.txn.ContainsKey(blockSpentTxesTableId, key)) { byte[] spentTxesBytes; using (var stream = new MemoryStream()) using (var writer = new BinaryWriter(stream)) { writer.WriteList(spentTxes, spentTx => DataEncoder.EncodeSpentTx(writer, spentTx)); spentTxesBytes = stream.ToArray(); } this.txn.Put(blockSpentTxesTableId, key, spentTxesBytes); return true; } else { return false; } }
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes) { CheckTransaction(); return this.blockSpentTxes.TryGetValue(blockIndex, out spentTxes); }
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes) { CheckTransaction(); return(blockSpentTxes.Value.TryGetValue(blockIndex, out spentTxes)); }
public static byte[] EncodeBlockSpentTxes(BlockSpentTxes blockSpentTxes) { using (var stream = new MemoryStream()) using (var writer = new BinaryWriter(stream)) { EncodeBlockSpentTxes(writer, blockSpentTxes); return stream.ToArray(); } }
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes) { CheckWriteTransaction(); try { this.blockSpentTxes.Add(blockIndex, spentTxes); this.blockSpentTxesModified = true; return true; } catch (ArgumentException) { return false; } }
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes) { return blockSpentTxes.TryAdd(blockIndex, spentTxes); }
public static void EncodeBlockSpentTxes(BinaryWriter writer, BlockSpentTxes blockSpentTxes) { writer.WriteList(blockSpentTxes, spentTx => EncodeSpentTx(writer, spentTx)); }
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes) { using (var handle = this.cursorCache.TakeItem()) { var cursor = handle.Item.Item; return cursor.TryGetBlockSpentTxes(blockIndex, out spentTxes); } }
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes) { CheckTransaction(); return(CursorTryGet(blockIndex, out spentTxes, this.spentTxes, MakeSpentTxesKey, x => DataDecoder.DecodeBlockSpentTxes(x))); }
private async Task PruneTxIndexAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes) { if (!mode.HasFlag(PruningMode.TxIndex)) return; var maxParallelism = Environment.ProcessorCount; // prepare a cache of cursors to be used by the pruning action block, allowing a pool of transactions var openedCursors = new ConcurrentBag<IChainStateCursor>(); using (var cursorHandles = new DisposableCache<DisposeHandle<IChainStateCursor>>(maxParallelism, () => { // retrieve a new cursor and start its transaction, keeping track of any cursors opened var cursorHandle = this.storageManager.OpenChainStateCursor(); cursorHandle.Item.BeginTransaction(pruneOnly: true); openedCursors.Add(cursorHandle.Item); return cursorHandle; })) { var pruneTxIndex = new ActionBlock<SpentTx>( spentTx => { using (var handle = cursorHandles.TakeItem()) { var chainStateCursor = handle.Item.Item; chainStateCursor.TryRemoveUnspentTx(spentTx.TxHash); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxParallelism }); var spentTxesQueue = new BufferBlock<SpentTx>(); spentTxesQueue.LinkTo(pruneTxIndex, new DataflowLinkOptions { PropagateCompletion = true }); await spentTxesQueue.SendAndCompleteAsync(spentTxes); await pruneTxIndex.Completion; // commit all opened cursors on success Parallel.ForEach(openedCursors, cursor => cursor.CommitTransaction()); } }
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes) { CheckTransaction(); using (SetSessionContext()) { Api.JetSetCurrentIndex(this.jetSession, this.spentTxTableId, "IX_SpentBlockIndex"); Api.MakeKey(this.jetSession, this.spentTxTableId, blockIndex, MakeKeyGrbit.NewKey); if (Api.TrySeek(this.jetSession, this.spentTxTableId, SeekGrbit.SeekEQ)) { var spentTxesBytes = Api.RetrieveColumn(this.jetSession, this.spentTxTableId, this.spentDataColumnId); spentTxes = DataDecoder.DecodeBlockSpentTxes(spentTxesBytes); return true; } else { spentTxes = null; return false; } } }
private async Task PruneBlockTxesAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes) { if (!mode.HasFlag(PruningMode.BlockTxesPreserveMerkle) && !mode.HasFlag(PruningMode.BlockTxesDestroyMerkle)) return; // create a source of txes to prune sources, for each block var pruningQueue = new BufferBlock<Tuple<int, List<int>>>(); // prepare tx pruner, to prune a txes source for a given block var txPruner = new ActionBlock<Tuple<int, List<int>>>( blockWorkItem => { var blockIndex = blockWorkItem.Item1; var blockHash = chain.Blocks[blockIndex].Hash; var spentTxIndices = blockWorkItem.Item2; var pruneWorkItem = new KeyValuePair<UInt256, IEnumerable<int>>(blockHash, spentTxIndices); if (mode.HasFlag(PruningMode.BlockTxesPreserveMerkle)) this.storageManager.BlockTxesStorage.PruneElements(new[] { pruneWorkItem }); else this.storageManager.BlockTxesStorage.DeleteElements(new[] { pruneWorkItem }); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }); pruningQueue.LinkTo(txPruner, new DataflowLinkOptions { PropagateCompletion = true }); // queue spent txes, grouped by block await pruningQueue.SendAndCompleteAsync( spentTxes.ReadByBlock().Select( spentTxesByBlock => { var blockIndex = spentTxesByBlock.Item1; var txIndices = spentTxesByBlock.Item2.Select(x => x.TxIndex).ToList(); return Tuple.Create(blockIndex, txIndices); })); await txPruner.Completion; }
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes) { CheckWriteTransaction(); return blockSpentTxes.TryAdd(blockIndex, spentTxes); }
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes) { CheckWriteTransaction(); return(blockSpentTxes.TryAdd(blockIndex, spentTxes)); }
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes) { CheckWriteTransaction(); try { blockSpentTxes.Modify(x => x.Add(blockIndex, spentTxes)); return true; } catch (ArgumentException) { return false; } }
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes) { CheckWriteTransaction(); try { using (SetSessionContext()) using (var jetUpdate = this.jetSession.BeginUpdate(this.spentTxTableId, JET_prep.Insert)) { byte[] spentTxesBytes; using (var stream = new MemoryStream()) using (var writer = new BinaryWriter(stream)) { writer.WriteList<SpentTx>(spentTxes, spentTx => DataEncoder.EncodeSpentTx(writer, spentTx)); spentTxesBytes = stream.ToArray(); } Api.SetColumns(this.jetSession, this.spentTxTableId, new Int32ColumnValue { Columnid = this.spentSpentBlockIndexColumnId, Value = blockIndex }, new BytesColumnValue { Columnid = this.spentDataColumnId, Value = spentTxesBytes }); jetUpdate.Save(); } return true; } catch (EsentKeyDuplicateException) { return false; } }
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes) { CheckTransaction(); byte[] spentTxesBytes; if (this.txn.TryGet(blockSpentTxesTableId, DbEncoder.EncodeInt32(blockIndex), out spentTxesBytes)) { using (var stream = new MemoryStream(spentTxesBytes)) using (var reader = new BinaryReader(stream)) { spentTxes = BlockSpentTxes.CreateRange(reader.ReadList(() => DataEncoder.DecodeSpentTx(reader))); } return true; } else { spentTxes = null; return false; } }
private async Task PruneBlockTxesAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes) { if (mode.HasFlag(PruningMode.BlockTxesDelete)) { storageManager.BlockTxesStorage.TryRemoveBlockTransactions(pruneBlock.Hash); } else if (mode.HasFlag(PruningMode.BlockTxesPreserveMerkle) || mode.HasFlag(PruningMode.BlockTxesDestroyMerkle)) { // create a source of txes to prune sources, for each block var pruningQueue = new BufferBlock <Tuple <int, List <int> > >(); // prepare tx pruner, to prune a txes source for a given block var txPruner = new ActionBlock <Tuple <int, List <int> > >( blockWorkItem => { var blockIndex = blockWorkItem.Item1; var blockHash = chain.Blocks[blockIndex].Hash; var spentTxIndices = blockWorkItem.Item2; var pruneWorkItem = new KeyValuePair <UInt256, IEnumerable <int> >(blockHash, spentTxIndices); if (mode.HasFlag(PruningMode.BlockTxesPreserveMerkle)) { this.storageManager.BlockTxesStorage.PruneElements(new[] { pruneWorkItem }); } else { this.storageManager.BlockTxesStorage.DeleteElements(new[] { pruneWorkItem }); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }); pruningQueue.LinkTo(txPruner, new DataflowLinkOptions { PropagateCompletion = true }); // queue spent txes, grouped by block await pruningQueue.SendAndCompleteAsync( spentTxes.ReadByBlock().Select( spentTxesByBlock => { var blockIndex = spentTxesByBlock.Item1; var txIndices = spentTxesByBlock.Item2.Select(x => x.TxIndex).ToList(); return(Tuple.Create(blockIndex, txIndices)); })); await txPruner.Completion; } }
private async Task PruneTxIndexAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes) { if (!mode.HasFlag(PruningMode.TxIndex)) { return; } var maxParallelism = Environment.ProcessorCount; // prepare a cache of cursors to be used by the pruning action block, allowing a pool of transactions var openedCursors = new ConcurrentBag <IChainStateCursor>(); using (var cursorHandles = new DisposableCache <DisposeHandle <IChainStateCursor> >(maxParallelism, () => { // retrieve a new cursor and start its transaction, keeping track of any cursors opened var cursorHandle = this.storageManager.OpenChainStateCursor(); cursorHandle.Item.BeginTransaction(); openedCursors.Add(cursorHandle.Item); return(cursorHandle); })) { var pruneTxIndex = new ActionBlock <SpentTx>( spentTx => { using (var handle = cursorHandles.TakeItem()) { var chainStateCursor = handle.Item.Item; chainStateCursor.RemoveUnspentTx(spentTx.TxHash); for (var outputIndex = 0; outputIndex < spentTx.OutputCount; outputIndex++) { chainStateCursor.RemoveUnspentTxOutput(new TxOutputKey(spentTx.TxHash, (uint)outputIndex)); } } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxParallelism }); var spentTxesQueue = new BufferBlock <SpentTx>(); spentTxesQueue.LinkTo(pruneTxIndex, new DataflowLinkOptions { PropagateCompletion = true }); await spentTxesQueue.SendAndCompleteAsync(spentTxes); await pruneTxIndex.Completion; // commit all opened cursors on success var commitTasks = new Task[openedCursors.Count]; var i = 0; foreach (var cursor in openedCursors) { commitTasks[i++] = cursor.CommitTransactionAsync(); } await Task.WhenAll(commitTasks); } }
public void TestPruneAllData() { // create genesis block var genesisblock = CreateFakeBlock(1); var genesisHeader = new ChainedHeader(genesisblock.Header, height: 0, totalWork: genesisblock.Header.CalculateWork().ToBigInteger(), dateSeen: DateTimeOffset.Now); // create a block var txCount = 100; var block = CreateFakeBlock(txCount, genesisblock.Hash); var chainedHeader = ChainedHeader.CreateFromPrev(genesisHeader, block.Header, dateSeen: DateTimeOffset.Now); // create a long chain based off the block, to account for pruning buffer var fakeHeaders = new FakeHeaders(new[] { genesisHeader, chainedHeader }); var chain = new ChainBuilder(Enumerable.Concat(new[] { genesisHeader, chainedHeader }, Enumerable.Range(0, 2000).Select(x => fakeHeaders.NextChained()))).ToImmutable(); // mock core daemon to return the chain var coreDaemon = new Mock <ICoreDaemon>(); coreDaemon.Setup(x => x.CurrentChain).Returns(chain); // create memory storage with the block var storageManager = new MemoryStorageManager(); storageManager.BlockTxesStorage.TryAddBlockTransactions(block.Hash, block.BlockTxes); // initialize the pruning worker var workerConfig = new WorkerConfig(initialNotify: false, minIdleTime: TimeSpan.Zero, maxIdleTime: TimeSpan.MaxValue); using (var pruningWorker = new PruningWorker(workerConfig, coreDaemon.Object, storageManager, null)) // get a chain state cursor using (var handle = storageManager.OpenChainStateCursor()) { var chainStateCursor = handle.Item; // set the pruning worker to prune all data pruningWorker.Mode = PruningMode.TxIndex | PruningMode.BlockSpentIndex | PruningMode.BlockTxesPreserveMerkle; // wire event to wait for work var workFinishedEvent = new AutoResetEvent(false); pruningWorker.OnWorkFinished += () => workFinishedEvent.Set(); // wire event to track exceptions Exception workException = null; pruningWorker.OnWorkError += e => workException = e; // start the worker pruningWorker.Start(); // pick a random pruning order var random = new Random(); var pruneOrderSource = Enumerable.Range(0, txCount).ToList(); var pruneOrder = new List <int>(txCount); while (pruneOrderSource.Count > 0) { var randomIndex = random.Next(pruneOrderSource.Count); pruneOrder.Add(pruneOrderSource[randomIndex]); pruneOrderSource.RemoveAt(randomIndex); } // add an unspent tx for each transaction to storage var unspentTxes = new UnspentTx[block.Transactions.Length]; chainStateCursor.BeginTransaction(); for (var txIndex = 0; txIndex < block.Transactions.Length; txIndex++) { var tx = block.Transactions[txIndex]; var unspentTx = new UnspentTx(tx.Hash, blockIndex: 1, txIndex: txIndex, txVersion: tx.Version, isCoinbase: txIndex == 0, outputStates: new OutputStates(1, OutputState.Spent)); unspentTxes[txIndex] = unspentTx; chainStateCursor.TryAddUnspentTx(unspentTx); } chainStateCursor.CommitTransaction(); // create a memory pruning cursor to verify expected pruning results var pruneCursor = new MemoryMerkleTreePruningCursor <BlockTxNode>(block.BlockTxes.Select(x => (BlockTxNode)x)); // prune each transaction in random order var pruneHeight = 0; foreach (var pruneTxIndex in pruneOrder) { // create a spent tx to prune the transaction var pruneTx = block.Transactions[pruneTxIndex]; var spentTxes = BlockSpentTxes.CreateRange(new[] { unspentTxes[pruneTxIndex].ToSpentTx() }); // store the spent txes for the current pruning block pruneHeight++; chainStateCursor.BeginTransaction(); Assert.IsTrue(chainStateCursor.TryAddBlockSpentTxes(pruneHeight, spentTxes)); pruningWorker.PrunableHeight = pruneHeight; // verify unspent tx is present before pruning Assert.IsTrue(chainStateCursor.ContainsUnspentTx(pruneTx.Hash)); chainStateCursor.CommitTransaction(); // notify the pruning worker and wait pruningWorker.NotifyWork(); workFinishedEvent.WaitOne(); // verify unspent tx is removed after pruning chainStateCursor.BeginTransaction(); Assert.IsFalse(chainStateCursor.ContainsUnspentTx(pruneTx.Hash)); // verify unspent tx outputs are removed after pruning for (var outputIndex = 0; outputIndex < pruneTx.Outputs.Length; outputIndex++) { Assert.IsFalse(chainStateCursor.ContainsUnspentTxOutput(new TxOutputKey(pruneTx.Hash, (uint)outputIndex))); } // verify the spent txes were removed Assert.IsFalse(chainStateCursor.ContainsBlockSpentTxes(pruneHeight)); chainStateCursor.RollbackTransaction(); // prune to determine expected results MerkleTree.PruneNode(pruneCursor, pruneTxIndex); var expectedPrunedTxes = pruneCursor.ReadNodes().ToList(); // retrieve the actual transaction after pruning IEnumerator <BlockTxNode> actualPrunedTxNodes; Assert.IsTrue(storageManager.BlockTxesStorage.TryReadBlockTxNodes(block.Hash, out actualPrunedTxNodes)); // verify the actual pruned transactions match the expected results CollectionAssert.AreEqual(expectedPrunedTxes, actualPrunedTxNodes.UsingAsEnumerable().ToList()); } // verify all unspent txes were removed chainStateCursor.BeginTransaction(); Assert.AreEqual(0, chainStateCursor.ReadUnspentTransactions().Count()); chainStateCursor.CommitTransaction(); // verify final block with all transactions pruned IEnumerator <BlockTxNode> finalPrunedTxNodes; Assert.IsTrue(storageManager.BlockTxesStorage.TryReadBlockTxNodes(block.Hash, out finalPrunedTxNodes)); var finalPrunedTxesList = finalPrunedTxNodes.UsingAsEnumerable().ToList(); Assert.AreEqual(1, finalPrunedTxesList.Count); Assert.AreEqual(block.Header.MerkleRoot, finalPrunedTxesList.Single().Hash); // verify no work exceptions occurred Assert.IsNull(workException); } }
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes) { return blockSpentTxes.TryGetValue(blockIndex, out spentTxes); }