private async Task PruneBlockTxesAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes)
{
if (mode.HasFlag(PruningMode.BlockTxesDelete))
{
storageManager.BlockTxesStorage.TryRemoveBlockTransactions(pruneBlock.Hash);
}
else if (mode.HasFlag(PruningMode.BlockTxesPreserveMerkle) || mode.HasFlag(PruningMode.BlockTxesDestroyMerkle))
{
// create a source of txes to prune sources, for each block
var pruningQueue = new BufferBlock <Tuple <int, List <int> > >();
// prepare tx pruner, to prune a txes source for a given block
var txPruner = new ActionBlock <Tuple <int, List <int> > >(
blockWorkItem =>
{
var blockIndex = blockWorkItem.Item1;
var blockHash = chain.Blocks[blockIndex].Hash;
var spentTxIndices = blockWorkItem.Item2;
var pruneWorkItem = new KeyValuePair <UInt256, IEnumerable <int> >(blockHash, spentTxIndices);
if (mode.HasFlag(PruningMode.BlockTxesPreserveMerkle))
{
this.storageManager.BlockTxesStorage.PruneElements(new[] { pruneWorkItem });
}
else
{
this.storageManager.BlockTxesStorage.DeleteElements(new[] { pruneWorkItem });
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
});
pruningQueue.LinkTo(txPruner, new DataflowLinkOptions {
PropagateCompletion = true
});
// queue spent txes, grouped by block
await pruningQueue.SendAndCompleteAsync(
spentTxes.ReadByBlock().Select(
spentTxesByBlock =>
{
var blockIndex = spentTxesByBlock.Item1;
var txIndices = spentTxesByBlock.Item2.Select(x => x.TxIndex).ToList();
return(Tuple.Create(blockIndex, txIndices));
}));
await txPruner.Completion;
}
}
private async Task PruneBlockSpentTxes(PruningMode mode, Chain chain, ChainedHeader pruneBlock)
{
if (!mode.HasFlag(PruningMode.BlockSpentIndex))
{
return;
}
using (var handle = this.storageManager.OpenChainStateCursor())
{
var chainStateCursor = handle.Item;
chainStateCursor.BeginTransaction();
// TODO don't immediately remove list of spent txes per block from chain state,
// use an additional safety buffer in case there was an issue pruning block
// txes (e.g. didn't flush and crashed), keeping the information will allow
// the block txes pruning to be performed again
chainStateCursor.TryRemoveBlockSpentTxes(pruneBlock.Height);
await chainStateCursor.CommitTransactionAsync();
}
}
private async Task PruneTxIndexAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes)
{
if (!mode.HasFlag(PruningMode.TxIndex))
{
return;
}
var maxParallelism = Environment.ProcessorCount;
// prepare a cache of cursors to be used by the pruning action block, allowing a pool of transactions
var openedCursors = new ConcurrentBag <IChainStateCursor>();
using (var cursorHandles = new DisposableCache <DisposeHandle <IChainStateCursor> >(maxParallelism,
() =>
{
// retrieve a new cursor and start its transaction, keeping track of any cursors opened
var cursorHandle = this.storageManager.OpenChainStateCursor();
cursorHandle.Item.BeginTransaction();
openedCursors.Add(cursorHandle.Item);
return(cursorHandle);
}))
{
var pruneTxIndex = new ActionBlock <SpentTx>(
spentTx =>
{
using (var handle = cursorHandles.TakeItem())
{
var chainStateCursor = handle.Item.Item;
chainStateCursor.RemoveUnspentTx(spentTx.TxHash);
for (var outputIndex = 0; outputIndex < spentTx.OutputCount; outputIndex++)
{
chainStateCursor.RemoveUnspentTxOutput(new TxOutputKey(spentTx.TxHash, (uint)outputIndex));
}
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxParallelism
});
var spentTxesQueue = new BufferBlock <SpentTx>();
spentTxesQueue.LinkTo(pruneTxIndex, new DataflowLinkOptions {
PropagateCompletion = true
});
await spentTxesQueue.SendAndCompleteAsync(spentTxes);
await pruneTxIndex.Completion;
// commit all opened cursors on success
var commitTasks = new Task[openedCursors.Count];
var i = 0;
foreach (var cursor in openedCursors)
{
commitTasks[i++] = cursor.CommitTransactionAsync();
}
await Task.WhenAll(commitTasks);
}
}
public static bool IsFull(this PruningMode mode) => (mode & PruningMode.Full) == PruningMode.Full;
private async Task PruneBlock(PruningMode mode, Chain chain, ChainedHeader pruneBlock)
{
//TODO the replay information about blocks that have been rolled back also needs to be pruned (UnmintedTx)
var txCount = 0;
var totalStopwatch = Stopwatch.StartNew();
var pruneBlockTxesStopwatch = new Stopwatch();
var pruneTxIndexStopwatch = new Stopwatch();
var pruneSpentTxesStopwatch = new Stopwatch();
// retrieve the spent txes for this block
BlockSpentTxes spentTxes;
using (var handle = this.storageManager.OpenChainStateCursor())
{
var chainStateCursor = handle.Item;
chainStateCursor.BeginTransaction(readOnly: true);
chainStateCursor.TryGetBlockSpentTxes(pruneBlock.Height, out spentTxes);
}
if (spentTxes != null)
{
txCount = spentTxes.Count;
pruneBlockTxesStopwatch.Start();
pruneTxIndexStopwatch.Start();
await Task.WhenAll(
// prune block txes (either merkle prune or delete)
PruneBlockTxesAsync(mode, chain, pruneBlock, spentTxes)
.ContinueWith(_ => pruneBlockTxesStopwatch.Stop()),
// prune tx index
PruneTxIndexAsync(mode, chain, pruneBlock, spentTxes)
.ContinueWith(_ => pruneTxIndexStopwatch.Stop())
);
// remove block spent txes information
//TODO should have a buffer on removing this, block txes pruning may need it again if flush doesn't happen
var pruneSpentTxesTask = PruneBlockSpentTxes(mode, chain, pruneBlock);
await pruneSpentTxesStopwatch.TimeAsync(pruneSpentTxesTask);
}
// if spent txes aren't available, block txes can still be deleted entirely for that pruning style
else if (mode.HasFlag(PruningMode.BlockTxesDelete))
{
pruneBlockTxesStopwatch.Start();
await PruneBlockTxesAsync(mode, chain, pruneBlock, null);
pruneBlockTxesStopwatch.Stop();
}
else //if (pruneBlock.Height > 0)
{
//TODO can't throw an exception unless the pruned chain is persisted
//logger.Info("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX: {0:N0}".Format2(pruneBlock.Height));
//throw new InvalidOperationException();
txCount = 0;
}
// track stats
txCountMeasure.Tick(txCount);
txRateMeasure.Tick((float)(txCount / totalStopwatch.Elapsed.TotalSeconds));
pruneBlockTxesDurationMeasure.Tick(pruneBlockTxesStopwatch.Elapsed);
pruneTxIndexDurationMeasure.Tick(pruneTxIndexStopwatch.Elapsed);
pruneSpentTxesDurationMeasure.Tick(pruneSpentTxesStopwatch.Elapsed);
totalDurationMeasure.Tick(totalStopwatch.Elapsed);
}
public static bool IsMemory(this PruningMode mode) => (mode & PruningMode.Memory) == PruningMode.Memory;
private void PruneBlockSpentTxes(PruningMode mode, Chain chain, ChainedHeader pruneBlock)
{
if (!mode.HasFlag(PruningMode.BlockSpentIndex))
return;
using (var handle = this.storageManager.OpenChainStateCursor())
{
var chainStateCursor = handle.Item;
chainStateCursor.BeginTransaction(pruneOnly: true);
// TODO don't immediately remove list of spent txes per block from chain state,
// use an additional safety buffer in case there was an issue pruning block
// txes (e.g. didn't flush and crashed), keeping the information will allow
// the block txes pruning to be performed again
chainStateCursor.TryRemoveBlockSpentTxes(pruneBlock.Height);
chainStateCursor.CommitTransaction();
}
}
private async Task PruneTxIndexAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes)
{
if (!mode.HasFlag(PruningMode.TxIndex))
return;
var maxParallelism = Environment.ProcessorCount;
// prepare a cache of cursors to be used by the pruning action block, allowing a pool of transactions
var openedCursors = new ConcurrentBag<IChainStateCursor>();
using (var cursorHandles = new DisposableCache<DisposeHandle<IChainStateCursor>>(maxParallelism,
() =>
{
// retrieve a new cursor and start its transaction, keeping track of any cursors opened
var cursorHandle = this.storageManager.OpenChainStateCursor();
cursorHandle.Item.BeginTransaction(pruneOnly: true);
openedCursors.Add(cursorHandle.Item);
return cursorHandle;
}))
{
var pruneTxIndex = new ActionBlock<SpentTx>(
spentTx =>
{
using (var handle = cursorHandles.TakeItem())
{
var chainStateCursor = handle.Item.Item;
chainStateCursor.TryRemoveUnspentTx(spentTx.TxHash);
}
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxParallelism });
var spentTxesQueue = new BufferBlock<SpentTx>();
spentTxesQueue.LinkTo(pruneTxIndex, new DataflowLinkOptions { PropagateCompletion = true });
await spentTxesQueue.SendAndCompleteAsync(spentTxes);
await pruneTxIndex.Completion;
// commit all opened cursors on success
Parallel.ForEach(openedCursors, cursor =>
cursor.CommitTransaction());
}
}
private async Task PruneBlockTxesAsync(PruningMode mode, Chain chain, ChainedHeader pruneBlock, BlockSpentTxes spentTxes)
{
if (!mode.HasFlag(PruningMode.BlockTxesPreserveMerkle) && !mode.HasFlag(PruningMode.BlockTxesDestroyMerkle))
return;
// create a source of txes to prune sources, for each block
var pruningQueue = new BufferBlock<Tuple<int, List<int>>>();
// prepare tx pruner, to prune a txes source for a given block
var txPruner = new ActionBlock<Tuple<int, List<int>>>(
blockWorkItem =>
{
var blockIndex = blockWorkItem.Item1;
var blockHash = chain.Blocks[blockIndex].Hash;
var spentTxIndices = blockWorkItem.Item2;
var pruneWorkItem = new KeyValuePair<UInt256, IEnumerable<int>>(blockHash, spentTxIndices);
if (mode.HasFlag(PruningMode.BlockTxesPreserveMerkle))
this.storageManager.BlockTxesStorage.PruneElements(new[] { pruneWorkItem });
else
this.storageManager.BlockTxesStorage.DeleteElements(new[] { pruneWorkItem });
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount });
pruningQueue.LinkTo(txPruner, new DataflowLinkOptions { PropagateCompletion = true });
// queue spent txes, grouped by block
await pruningQueue.SendAndCompleteAsync(
spentTxes.ReadByBlock().Select(
spentTxesByBlock =>
{
var blockIndex = spentTxesByBlock.Item1;
var txIndices = spentTxesByBlock.Item2.Select(x => x.TxIndex).ToList();
return Tuple.Create(blockIndex, txIndices);
}));
await txPruner.Completion;
}
private async Task PruneBlock(PruningMode mode, Chain chain, ChainedHeader pruneBlock)
{
//TODO the replay information about blocks that have been rolled back also needs to be pruned (UnmintedTx)
var txCount = 0;
var totalStopwatch = Stopwatch.StartNew();
var pruneBlockTxesStopwatch = new Stopwatch();
var pruneTxIndexStopwatch = new Stopwatch();
var pruneSpentTxesStopwatch = new Stopwatch();
// retrieve the spent txes for this block
BlockSpentTxes spentTxes;
using (var handle = this.storageManager.OpenChainStateCursor())
{
var chainStateCursor = handle.Item;
chainStateCursor.BeginTransaction(readOnly: true);
chainStateCursor.TryGetBlockSpentTxes(pruneBlock.Height, out spentTxes);
}
if (spentTxes != null)
{
txCount = spentTxes.Count;
pruneBlockTxesStopwatch.Start();
pruneTxIndexStopwatch.Start();
await Task.WhenAll(
// prune block txes (either merkle prune or delete)
PruneBlockTxesAsync(mode, chain, pruneBlock, spentTxes)
.ContinueWith(task => { pruneBlockTxesStopwatch.Stop(); task.Wait(); }),
// prune tx index
PruneTxIndexAsync(mode, chain, pruneBlock, spentTxes)
.ContinueWith(task => { pruneTxIndexStopwatch.Stop(); task.Wait(); })
);
// remove block spent txes information
//TODO should have a buffer on removing this, block txes pruning may need it again if flush doesn't happen
pruneSpentTxesStopwatch.Time(() =>
PruneBlockSpentTxes(mode, chain, pruneBlock));
}
else //if (pruneBlock.Height > 0)
{
//TODO can't throw an exception unless the pruned chain is persisted
//logger.Info("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX: {0:N0}".Format2(pruneBlock.Height));
//throw new InvalidOperationException();
txCount = 0;
}
// track stats
txCountMeasure.Tick(txCount);
txRateMeasure.Tick((float)(txCount / totalStopwatch.Elapsed.TotalSeconds));
pruneBlockTxesDurationMeasure.Tick(pruneBlockTxesStopwatch.Elapsed);
pruneTxIndexDurationMeasure.Tick(pruneTxIndexStopwatch.Elapsed);
pruneSpentTxesDurationMeasure.Tick(pruneSpentTxesStopwatch.Elapsed);
totalDurationMeasure.Tick(totalStopwatch.Elapsed);
}
