public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes)
{
CheckWriteTransaction();
try
{
using (SetSessionContext())
using (var jetUpdate = this.jetSession.BeginUpdate(this.spentTxTableId, JET_prep.Insert))
{
byte[] spentTxesBytes;
using (var stream = new MemoryStream())
using (var writer = new BinaryWriter(stream))
{
writer.WriteList <SpentTx>(spentTxes, spentTx => DataEncoder.EncodeSpentTx(writer, spentTx));
spentTxesBytes = stream.ToArray();
}
Api.SetColumns(this.jetSession, this.spentTxTableId,
new Int32ColumnValue {
Columnid = this.spentSpentBlockIndexColumnId, Value = blockIndex
},
new BytesColumnValue {
Columnid = this.spentDataColumnId, Value = spentTxesBytes
});
jetUpdate.Save();
}
return(true);
}
catch (EsentKeyDuplicateException)
{
return(false);
}
}
public static byte[] EncodeBlockSpentTxes(BlockSpentTxes blockSpentTxes)
{
using (var stream = new MemoryStream())
using (var writer = new BinaryWriter(stream))
{
EncodeBlockSpentTxes(writer, blockSpentTxes);
return(stream.ToArray());
}
}
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes)
{
CheckWriteTransaction();
try
{
blockSpentTxes.Modify(x => x.Add(blockIndex, spentTxes));
return(true);
}
catch (ArgumentException)
{
return(false);
}
}
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes)
{
CheckTransaction();
using (SetSessionContext())
{
Api.JetSetCurrentIndex(this.jetSession, this.spentTxTableId, "IX_SpentBlockIndex");
Api.MakeKey(this.jetSession, this.spentTxTableId, blockIndex, MakeKeyGrbit.NewKey);
if (Api.TrySeek(this.jetSession, this.spentTxTableId, SeekGrbit.SeekEQ))
{
var spentTxesBytes = Api.RetrieveColumn(this.jetSession, this.spentTxTableId, this.spentDataColumnId);
spentTxes = DataDecoder.DecodeBlockSpentTxes(spentTxesBytes);
return(true);
}
else
{
spentTxes = null;
return(false);
}
}
}
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes)
{
CheckWriteTransaction();
var key = DbEncoder.EncodeInt32(blockIndex);
if (!this.txn.ContainsKey(blockSpentTxesTableId, key))
{
byte[] spentTxesBytes;
using (var stream = new MemoryStream())
using (var writer = new BinaryWriter(stream))
{
writer.WriteList(spentTxes, spentTx => DataEncoder.EncodeSpentTx(writer, spentTx));
spentTxesBytes = stream.ToArray();
}
this.txn.Put(blockSpentTxesTableId, key, spentTxesBytes);
return true;
}
else
{
return false;
}
}
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes)
{
CheckTransaction();
return this.blockSpentTxes.TryGetValue(blockIndex, out spentTxes);
}
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes)
{
CheckTransaction();
return(blockSpentTxes.Value.TryGetValue(blockIndex, out spentTxes));
}
public static byte[] EncodeBlockSpentTxes(BlockSpentTxes blockSpentTxes)
{
using (var stream = new MemoryStream())
using (var writer = new BinaryWriter(stream))
{
EncodeBlockSpentTxes(writer, blockSpentTxes);
return stream.ToArray();
}
}
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes)
{
CheckWriteTransaction();
try
{
this.blockSpentTxes.Add(blockIndex, spentTxes);
this.blockSpentTxesModified = true;
return true;
}
catch (ArgumentException)
{
return false;
}
}
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes)
{
return blockSpentTxes.TryAdd(blockIndex, spentTxes);
}
public static void EncodeBlockSpentTxes(BinaryWriter writer, BlockSpentTxes blockSpentTxes)
{
writer.WriteList(blockSpentTxes, spentTx => EncodeSpentTx(writer, spentTx));
}
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes)
{
using (var handle = this.cursorCache.TakeItem())
{
var cursor = handle.Item.Item;
return cursor.TryGetBlockSpentTxes(blockIndex, out spentTxes);
}
}
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes)
{
CheckTransaction();
return(CursorTryGet(blockIndex, out spentTxes, this.spentTxes, MakeSpentTxesKey, x => DataDecoder.DecodeBlockSpentTxes(x)));
}
private async Task PruneTxIndexAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes)
{
if (!mode.HasFlag(PruningMode.TxIndex))
return;
var maxParallelism = Environment.ProcessorCount;
// prepare a cache of cursors to be used by the pruning action block, allowing a pool of transactions
var openedCursors = new ConcurrentBag<IChainStateCursor>();
using (var cursorHandles = new DisposableCache<DisposeHandle<IChainStateCursor>>(maxParallelism,
() =>
{
// retrieve a new cursor and start its transaction, keeping track of any cursors opened
var cursorHandle = this.storageManager.OpenChainStateCursor();
cursorHandle.Item.BeginTransaction(pruneOnly: true);
openedCursors.Add(cursorHandle.Item);
return cursorHandle;
}))
{
var pruneTxIndex = new ActionBlock<SpentTx>(
spentTx =>
{
using (var handle = cursorHandles.TakeItem())
{
var chainStateCursor = handle.Item.Item;
chainStateCursor.TryRemoveUnspentTx(spentTx.TxHash);
}
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxParallelism });
var spentTxesQueue = new BufferBlock<SpentTx>();
spentTxesQueue.LinkTo(pruneTxIndex, new DataflowLinkOptions { PropagateCompletion = true });
await spentTxesQueue.SendAndCompleteAsync(spentTxes);
await pruneTxIndex.Completion;
// commit all opened cursors on success
Parallel.ForEach(openedCursors, cursor =>
cursor.CommitTransaction());
}
}
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes)
{
CheckTransaction();
using (SetSessionContext())
{
Api.JetSetCurrentIndex(this.jetSession, this.spentTxTableId, "IX_SpentBlockIndex");
Api.MakeKey(this.jetSession, this.spentTxTableId, blockIndex, MakeKeyGrbit.NewKey);
if (Api.TrySeek(this.jetSession, this.spentTxTableId, SeekGrbit.SeekEQ))
{
var spentTxesBytes = Api.RetrieveColumn(this.jetSession, this.spentTxTableId, this.spentDataColumnId);
spentTxes = DataDecoder.DecodeBlockSpentTxes(spentTxesBytes);
return true;
}
else
{
spentTxes = null;
return false;
}
}
}
private async Task PruneBlockTxesAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes)
{
if (!mode.HasFlag(PruningMode.BlockTxesPreserveMerkle) && !mode.HasFlag(PruningMode.BlockTxesDestroyMerkle))
return;
// create a source of txes to prune sources, for each block
var pruningQueue = new BufferBlock<Tuple<int, List<int>>>();
// prepare tx pruner, to prune a txes source for a given block
var txPruner = new ActionBlock<Tuple<int, List<int>>>(
blockWorkItem =>
{
var blockIndex = blockWorkItem.Item1;
var blockHash = chain.Blocks[blockIndex].Hash;
var spentTxIndices = blockWorkItem.Item2;
var pruneWorkItem = new KeyValuePair<UInt256, IEnumerable<int>>(blockHash, spentTxIndices);
if (mode.HasFlag(PruningMode.BlockTxesPreserveMerkle))
this.storageManager.BlockTxesStorage.PruneElements(new[] { pruneWorkItem });
else
this.storageManager.BlockTxesStorage.DeleteElements(new[] { pruneWorkItem });
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount });
pruningQueue.LinkTo(txPruner, new DataflowLinkOptions { PropagateCompletion = true });
// queue spent txes, grouped by block
await pruningQueue.SendAndCompleteAsync(
spentTxes.ReadByBlock().Select(
spentTxesByBlock =>
{
var blockIndex = spentTxesByBlock.Item1;
var txIndices = spentTxesByBlock.Item2.Select(x => x.TxIndex).ToList();
return Tuple.Create(blockIndex, txIndices);
}));
await txPruner.Completion;
}
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes)
{
CheckWriteTransaction();
return blockSpentTxes.TryAdd(blockIndex, spentTxes);
}
public static void EncodeBlockSpentTxes(BinaryWriter writer, BlockSpentTxes blockSpentTxes)
{
writer.WriteList(blockSpentTxes, spentTx => EncodeSpentTx(writer, spentTx));
}
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes)
{
CheckWriteTransaction();
return(blockSpentTxes.TryAdd(blockIndex, spentTxes));
}
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes)
{
CheckWriteTransaction();
try
{
blockSpentTxes.Modify(x => x.Add(blockIndex, spentTxes));
return true;
}
catch (ArgumentException)
{
return false;
}
}
public bool TryAddBlockSpentTxes(int blockIndex, BlockSpentTxes spentTxes)
{
CheckWriteTransaction();
try
{
using (SetSessionContext())
using (var jetUpdate = this.jetSession.BeginUpdate(this.spentTxTableId, JET_prep.Insert))
{
byte[] spentTxesBytes;
using (var stream = new MemoryStream())
using (var writer = new BinaryWriter(stream))
{
writer.WriteList<SpentTx>(spentTxes, spentTx => DataEncoder.EncodeSpentTx(writer, spentTx));
spentTxesBytes = stream.ToArray();
}
Api.SetColumns(this.jetSession, this.spentTxTableId,
new Int32ColumnValue { Columnid = this.spentSpentBlockIndexColumnId, Value = blockIndex },
new BytesColumnValue { Columnid = this.spentDataColumnId, Value = spentTxesBytes });
jetUpdate.Save();
}
return true;
}
catch (EsentKeyDuplicateException)
{
return false;
}
}
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes)
{
CheckTransaction();
byte[] spentTxesBytes;
if (this.txn.TryGet(blockSpentTxesTableId, DbEncoder.EncodeInt32(blockIndex), out spentTxesBytes))
{
using (var stream = new MemoryStream(spentTxesBytes))
using (var reader = new BinaryReader(stream))
{
spentTxes = BlockSpentTxes.CreateRange(reader.ReadList(() => DataEncoder.DecodeSpentTx(reader)));
}
return true;
}
else
{
spentTxes = null;
return false;
}
}
private async Task PruneBlockTxesAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes)
{
if (mode.HasFlag(PruningMode.BlockTxesDelete))
{
storageManager.BlockTxesStorage.TryRemoveBlockTransactions(pruneBlock.Hash);
}
else if (mode.HasFlag(PruningMode.BlockTxesPreserveMerkle) || mode.HasFlag(PruningMode.BlockTxesDestroyMerkle))
{
// create a source of txes to prune sources, for each block
var pruningQueue = new BufferBlock <Tuple <int, List <int> > >();
// prepare tx pruner, to prune a txes source for a given block
var txPruner = new ActionBlock <Tuple <int, List <int> > >(
blockWorkItem =>
{
var blockIndex = blockWorkItem.Item1;
var blockHash = chain.Blocks[blockIndex].Hash;
var spentTxIndices = blockWorkItem.Item2;
var pruneWorkItem = new KeyValuePair <UInt256, IEnumerable <int> >(blockHash, spentTxIndices);
if (mode.HasFlag(PruningMode.BlockTxesPreserveMerkle))
{
this.storageManager.BlockTxesStorage.PruneElements(new[] { pruneWorkItem });
}
else
{
this.storageManager.BlockTxesStorage.DeleteElements(new[] { pruneWorkItem });
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
});
pruningQueue.LinkTo(txPruner, new DataflowLinkOptions {
PropagateCompletion = true
});
// queue spent txes, grouped by block
await pruningQueue.SendAndCompleteAsync(
spentTxes.ReadByBlock().Select(
spentTxesByBlock =>
{
var blockIndex = spentTxesByBlock.Item1;
var txIndices = spentTxesByBlock.Item2.Select(x => x.TxIndex).ToList();
return(Tuple.Create(blockIndex, txIndices));
}));
await txPruner.Completion;
}
}
private async Task PruneTxIndexAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes)
{
if (!mode.HasFlag(PruningMode.TxIndex))
{
return;
}
var maxParallelism = Environment.ProcessorCount;
// prepare a cache of cursors to be used by the pruning action block, allowing a pool of transactions
var openedCursors = new ConcurrentBag <IChainStateCursor>();
using (var cursorHandles = new DisposableCache <DisposeHandle <IChainStateCursor> >(maxParallelism,
() =>
{
// retrieve a new cursor and start its transaction, keeping track of any cursors opened
var cursorHandle = this.storageManager.OpenChainStateCursor();
cursorHandle.Item.BeginTransaction();
openedCursors.Add(cursorHandle.Item);
return(cursorHandle);
}))
{
var pruneTxIndex = new ActionBlock <SpentTx>(
spentTx =>
{
using (var handle = cursorHandles.TakeItem())
{
var chainStateCursor = handle.Item.Item;
chainStateCursor.RemoveUnspentTx(spentTx.TxHash);
for (var outputIndex = 0; outputIndex < spentTx.OutputCount; outputIndex++)
{
chainStateCursor.RemoveUnspentTxOutput(new TxOutputKey(spentTx.TxHash, (uint)outputIndex));
}
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxParallelism
});
var spentTxesQueue = new BufferBlock <SpentTx>();
spentTxesQueue.LinkTo(pruneTxIndex, new DataflowLinkOptions {
PropagateCompletion = true
});
await spentTxesQueue.SendAndCompleteAsync(spentTxes);
await pruneTxIndex.Completion;
// commit all opened cursors on success
var commitTasks = new Task[openedCursors.Count];
var i = 0;
foreach (var cursor in openedCursors)
{
commitTasks[i++] = cursor.CommitTransactionAsync();
}
await Task.WhenAll(commitTasks);
}
}
public void TestPruneAllData()
{
// create genesis block
var genesisblock = CreateFakeBlock(1);
var genesisHeader = new ChainedHeader(genesisblock.Header, height: 0, totalWork: genesisblock.Header.CalculateWork().ToBigInteger(), dateSeen: DateTimeOffset.Now);
// create a block
var txCount = 100;
var block = CreateFakeBlock(txCount, genesisblock.Hash);
var chainedHeader = ChainedHeader.CreateFromPrev(genesisHeader, block.Header, dateSeen: DateTimeOffset.Now);
// create a long chain based off the block, to account for pruning buffer
var fakeHeaders = new FakeHeaders(new[] { genesisHeader, chainedHeader });
var chain = new ChainBuilder(Enumerable.Concat(new[] { genesisHeader, chainedHeader }, Enumerable.Range(0, 2000).Select(x => fakeHeaders.NextChained()))).ToImmutable();
// mock core daemon to return the chain
var coreDaemon = new Mock <ICoreDaemon>();
coreDaemon.Setup(x => x.CurrentChain).Returns(chain);
// create memory storage with the block
var storageManager = new MemoryStorageManager();
storageManager.BlockTxesStorage.TryAddBlockTransactions(block.Hash, block.BlockTxes);
// initialize the pruning worker
var workerConfig = new WorkerConfig(initialNotify: false, minIdleTime: TimeSpan.Zero, maxIdleTime: TimeSpan.MaxValue);
using (var pruningWorker = new PruningWorker(workerConfig, coreDaemon.Object, storageManager, null))
// get a chain state cursor
using (var handle = storageManager.OpenChainStateCursor())
{
var chainStateCursor = handle.Item;
// set the pruning worker to prune all data
pruningWorker.Mode = PruningMode.TxIndex | PruningMode.BlockSpentIndex | PruningMode.BlockTxesPreserveMerkle;
// wire event to wait for work
var workFinishedEvent = new AutoResetEvent(false);
pruningWorker.OnWorkFinished += () => workFinishedEvent.Set();
// wire event to track exceptions
Exception workException = null;
pruningWorker.OnWorkError += e => workException = e;
// start the worker
pruningWorker.Start();
// pick a random pruning order
var random = new Random();
var pruneOrderSource = Enumerable.Range(0, txCount).ToList();
var pruneOrder = new List <int>(txCount);
while (pruneOrderSource.Count > 0)
{
var randomIndex = random.Next(pruneOrderSource.Count);
pruneOrder.Add(pruneOrderSource[randomIndex]);
pruneOrderSource.RemoveAt(randomIndex);
}
// add an unspent tx for each transaction to storage
var unspentTxes = new UnspentTx[block.Transactions.Length];
chainStateCursor.BeginTransaction();
for (var txIndex = 0; txIndex < block.Transactions.Length; txIndex++)
{
var tx = block.Transactions[txIndex];
var unspentTx = new UnspentTx(tx.Hash, blockIndex: 1, txIndex: txIndex, txVersion: tx.Version, isCoinbase: txIndex == 0, outputStates: new OutputStates(1, OutputState.Spent));
unspentTxes[txIndex] = unspentTx;
chainStateCursor.TryAddUnspentTx(unspentTx);
}
chainStateCursor.CommitTransaction();
// create a memory pruning cursor to verify expected pruning results
var pruneCursor = new MemoryMerkleTreePruningCursor <BlockTxNode>(block.BlockTxes.Select(x => (BlockTxNode)x));
// prune each transaction in random order
var pruneHeight = 0;
foreach (var pruneTxIndex in pruneOrder)
{
// create a spent tx to prune the transaction
var pruneTx = block.Transactions[pruneTxIndex];
var spentTxes = BlockSpentTxes.CreateRange(new[] { unspentTxes[pruneTxIndex].ToSpentTx() });
// store the spent txes for the current pruning block
pruneHeight++;
chainStateCursor.BeginTransaction();
Assert.IsTrue(chainStateCursor.TryAddBlockSpentTxes(pruneHeight, spentTxes));
pruningWorker.PrunableHeight = pruneHeight;
// verify unspent tx is present before pruning
Assert.IsTrue(chainStateCursor.ContainsUnspentTx(pruneTx.Hash));
chainStateCursor.CommitTransaction();
// notify the pruning worker and wait
pruningWorker.NotifyWork();
workFinishedEvent.WaitOne();
// verify unspent tx is removed after pruning
chainStateCursor.BeginTransaction();
Assert.IsFalse(chainStateCursor.ContainsUnspentTx(pruneTx.Hash));
// verify unspent tx outputs are removed after pruning
for (var outputIndex = 0; outputIndex < pruneTx.Outputs.Length; outputIndex++)
{
Assert.IsFalse(chainStateCursor.ContainsUnspentTxOutput(new TxOutputKey(pruneTx.Hash, (uint)outputIndex)));
}
// verify the spent txes were removed
Assert.IsFalse(chainStateCursor.ContainsBlockSpentTxes(pruneHeight));
chainStateCursor.RollbackTransaction();
// prune to determine expected results
MerkleTree.PruneNode(pruneCursor, pruneTxIndex);
var expectedPrunedTxes = pruneCursor.ReadNodes().ToList();
// retrieve the actual transaction after pruning
IEnumerator <BlockTxNode> actualPrunedTxNodes;
Assert.IsTrue(storageManager.BlockTxesStorage.TryReadBlockTxNodes(block.Hash, out actualPrunedTxNodes));
// verify the actual pruned transactions match the expected results
CollectionAssert.AreEqual(expectedPrunedTxes, actualPrunedTxNodes.UsingAsEnumerable().ToList());
}
// verify all unspent txes were removed
chainStateCursor.BeginTransaction();
Assert.AreEqual(0, chainStateCursor.ReadUnspentTransactions().Count());
chainStateCursor.CommitTransaction();
// verify final block with all transactions pruned
IEnumerator <BlockTxNode> finalPrunedTxNodes;
Assert.IsTrue(storageManager.BlockTxesStorage.TryReadBlockTxNodes(block.Hash, out finalPrunedTxNodes));
var finalPrunedTxesList = finalPrunedTxNodes.UsingAsEnumerable().ToList();
Assert.AreEqual(1, finalPrunedTxesList.Count);
Assert.AreEqual(block.Header.MerkleRoot, finalPrunedTxesList.Single().Hash);
// verify no work exceptions occurred
Assert.IsNull(workException);
}
}
public bool TryGetBlockSpentTxes(int blockIndex, out BlockSpentTxes spentTxes)
{
return blockSpentTxes.TryGetValue(blockIndex, out spentTxes);
}
