public async Task PruneRepository_PruneAndCompact_FromGenesis_OnStartUpAsync()
{
var posBlocks = CreatePosBlocks(50);
var chainedHeaderTip = BuildProvenHeaderChainFromBlocks(posBlocks);
var dataFolderPath = CreateTestDir(this);
var dataFolder = new DataFolder(dataFolderPath);
var dBreezeSerializer = new DBreezeSerializer(this.Network);
var blockRepository = new BlockRepository(this.Network, dataFolder, this.LoggerFactory.Object, dBreezeSerializer);
await blockRepository.PutAsync(new HashHeightPair(posBlocks.Last().GetHash(), 50), posBlocks);
var storeSettings = new StoreSettings(NodeSettings.Default(this.Network))
{
AmountOfBlocksToKeep = 10
};
var prunedBlockRepository = new PrunedBlockRepository(blockRepository, dBreezeSerializer, this.LoggerFactory.Object, storeSettings);
await prunedBlockRepository.InitializeAsync();
await prunedBlockRepository.PruneAndCompactDatabase(chainedHeaderTip.GetAncestor(50), this.Network, true);
// The first prune will delete blocks from 40 to 0.
Assert.Equal(chainedHeaderTip.GetAncestor(40).HashBlock, prunedBlockRepository.PrunedTip.Hash);
Assert.Equal(chainedHeaderTip.GetAncestor(40).Height, prunedBlockRepository.PrunedTip.Height);
// Ensure that the block has been deleted from disk.
Assert.Null(await blockRepository.GetBlockAsync(chainedHeaderTip.GetAncestor(39).HashBlock));
}
public void DownloadBlocks_CanBreakExecutionOnStallCountReached()
{
// Create 3 blocks
List <Block> blocks = CreateBlocks(3);
// The repository has 3 blocks stored
using (var blockRepository = new BlockRepository(Network.Main, TestBase.AssureEmptyDirAsDataFolder(@"BlockStore\DownloadBlocks_EnsureNextChainedBlockIsAskedForOnStartUp")))
{
blockRepository.PutAsync(blocks.Last().GetHash(), blocks.Take(3).ToList()).GetAwaiter().GetResult();
// The chain has 3 blocks appended
var chain = new ConcurrentChain(blocks[0].Header);
AppendBlocks(chain, blocks.Skip(1).Take(2));
// Create block store loop
var blockStoreLoop = CreateBlockStoreLoop(chain, blockRepository, @"BlockStore\DownloadBlocks_EnsureNextChainedBlockIsAskedForOnStartUp");
//Start finding blocks from Block[1]
var nextChainedBlock = blockStoreLoop.Chain.GetBlock(blocks[1].GetHash());
// Create Task Context
var context = new BlockStoreInnerStepContext(new CancellationToken(), blockStoreLoop).Initialize(nextChainedBlock);
context.StallCount = 10001;
var task = new BlockStoreInnerStepDownloadBlocks();
var result = task.ExecuteAsync(context).GetAwaiter().GetResult();
Assert.True(result.ShouldBreak);
}
}
public async Task PruneRepository_PruneAndCompact_OnShutDownAsync()
{
var posBlocks = CreatePosBlocks(50);
var chainedHeaderTip = BuildProvenHeaderChainFromBlocks(posBlocks);
var dataFolderPath = CreateTestDir(this);
var dataFolder = new DataFolder(dataFolderPath);
var dBreezeSerializer = new DBreezeSerializer(this.Network);
var blockRepository = new BlockRepository(this.Network, dataFolder, this.LoggerFactory.Object, dBreezeSerializer);
await blockRepository.PutAsync(new HashHeightPair(posBlocks.Last().GetHash(), 50), posBlocks);
var storeSettings = new StoreSettings(NodeSettings.Default(this.Network))
{
AmountOfBlocksToKeep = 10
};
var prunedBlockRepository = new PrunedBlockRepository(blockRepository, dBreezeSerializer, this.LoggerFactory.Object, storeSettings);
await prunedBlockRepository.InitializeAsync();
// Delete blocks 30 to 0 from the repo, this would have been done by the service before shutdown was initiated.
await blockRepository.DeleteBlocksAsync(posBlocks.Take(30).Select(b => b.GetHash()).ToList());
prunedBlockRepository.UpdatePrunedTip(chainedHeaderTip.GetAncestor(30));
// Ensure that the block has been deleted from disk.
Assert.Null(await blockRepository.GetBlockAsync(chainedHeaderTip.GetAncestor(29).HashBlock));
// On shutdown the database will only be compacted.
await prunedBlockRepository.PruneAndCompactDatabase(chainedHeaderTip.GetAncestor(50), this.Network, false);
Assert.Equal(chainedHeaderTip.GetAncestor(30).HashBlock, prunedBlockRepository.PrunedTip.Hash);
Assert.Equal(chainedHeaderTip.GetAncestor(30).Height, prunedBlockRepository.PrunedTip.Height);
Assert.Null(await blockRepository.GetBlockAsync(chainedHeaderTip.GetAncestor(29).HashBlock));
}
public void CanExecute_TryPending()
{
// Create 15 blocks
var blocks = CreateBlocks(15);
using (var blockRepository = new BlockRepository(Network.Main, TestBase.AssureEmptyDirAsDataFolder(@"BlockStore\CanExecute_TryPending")))
{
// Push 5 blocks to the repository
blockRepository.PutAsync(blocks.Take(5).Last().GetHash(), blocks.Take(5).ToList()).GetAwaiter().GetResult();
// The chain has 10 blocks appended
var chain = new ConcurrentChain(blocks[0].Header);
AppendBlocks(chain, blocks.Skip(1).Take(9));
// Create block store loop
var blockStoreLoop = CreateBlockStoreLoop(chain, blockRepository, @"BlockStore\CanExecute_TryPending");
// Add chained blocks 5 - 9 to PendingStorage
AddToPendingStorage(blockStoreLoop, blocks[5]);
AddToPendingStorage(blockStoreLoop, blocks[6]);
AddToPendingStorage(blockStoreLoop, blocks[7]);
AddToPendingStorage(blockStoreLoop, blocks[8]);
AddToPendingStorage(blockStoreLoop, blocks[9]);
//Start processing pending blocks from block 5
var nextChainedBlock = blockStoreLoop.Chain.GetBlock(blocks[5].GetHash());
var processPendingStorageStep = new ProcessPendingStorageStep(blockStoreLoop);
processPendingStorageStep.ExecuteAsync(nextChainedBlock, new CancellationToken(), false).GetAwaiter().GetResult();
Assert.Equal(blocks[9].GetHash(), blockStoreLoop.BlockRepository.BlockHash);
Assert.Equal(blocks[9].GetHash(), blockStoreLoop.StoreTip.HashBlock);
}
}
public void CanExecute_CheckNextChainedBlockExistStep()
{
var blocks = CreateBlocks(5);
using (var blockRepository = new BlockRepository(Network.Main, TestBase.AssureEmptyDirAsDataFolder(@"BlockStore\CanExecute_CheckNextChainedBlockExistStep")))
{
// Push 5 blocks to the repository
blockRepository.PutAsync(blocks.Last().GetHash(), blocks).GetAwaiter().GetResult();
// The chain has 4 blocks appended
var chain = new ConcurrentChain(blocks[0].Header);
AppendBlocks(chain, blocks.Skip(1).Take(3));
// Create the last chained block without appending to the chain
var block03 = chain.GetBlock(blocks[3].GetHash());
var block04 = new ChainedBlock(blocks[4].Header, blocks[4].Header.GetHash(), block03);
var blockStoreLoop = CreateBlockStoreLoop(chain, blockRepository, @"BlockStore\CanExecute_CheckNextChainedBlockExistStep");
Assert.Null(blockStoreLoop.StoreTip);
var nextChainedBlock = block04;
var checkExistsStep = new CheckNextChainedBlockExistStep(blockStoreLoop);
checkExistsStep.ExecuteAsync(nextChainedBlock, new CancellationToken(), false).GetAwaiter().GetResult();
Assert.Equal(blockStoreLoop.StoreTip.Header.GetHash(), block04.Header.GetHash());
Assert.Equal(blockStoreLoop.BlockRepository.BlockHash, block04.Header.GetHash());
}
}
private void BlockRepositoryBench()
{
using (var dir = TestDirectory.Create())
{
using (var blockRepo = new BlockRepository(Network.Main, dir.FolderName, DateTimeProvider.Default, this.loggerFactory))
{
var lst = new List <Block>();
for (int i = 0; i < 30; i++)
{
// roughly 1mb blocks
var block = new Block();
for (int j = 0; j < 3000; j++)
{
var trx = new Transaction();
block.AddTransaction(new Transaction());
trx.AddInput(new TxIn(Script.Empty));
trx.AddOutput(Money.COIN + j + i, new Script(Guid.NewGuid().ToByteArray()
.Concat(Guid.NewGuid().ToByteArray())
.Concat(Guid.NewGuid().ToByteArray())
.Concat(Guid.NewGuid().ToByteArray())
.Concat(Guid.NewGuid().ToByteArray())
.Concat(Guid.NewGuid().ToByteArray())));
trx.AddInput(new TxIn(Script.Empty));
trx.AddOutput(Money.COIN + j + i + 1, new Script(Guid.NewGuid().ToByteArray()
.Concat(Guid.NewGuid().ToByteArray())
.Concat(Guid.NewGuid().ToByteArray())
.Concat(Guid.NewGuid().ToByteArray())
.Concat(Guid.NewGuid().ToByteArray())
.Concat(Guid.NewGuid().ToByteArray())));
block.AddTransaction(trx);
}
block.UpdateMerkleRoot();
block.Header.HashPrevBlock = lst.Any() ? lst.Last().GetHash() : Network.Main.GenesisHash;
lst.Add(block);
}
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
blockRepo.PutAsync(lst.Last().GetHash(), lst).GetAwaiter().GetResult();
var first = stopwatch.ElapsedMilliseconds;
blockRepo.PutAsync(lst.Last().GetHash(), lst).GetAwaiter().GetResult();
var second = stopwatch.ElapsedMilliseconds;
}
}
}
public void FindBlocks_CanFind()
{
// Create 10 blocks
List <Block> blocks = CreateBlocks(10);
using (var blockRepository = new BlockRepository(Network.Main, TestBase.AssureEmptyDirAsDataFolder(@"BlockStore\FindBlocks_CanFind")))
{
// Push 5 blocks to the repository
blockRepository.PutAsync(blocks.Take(5).Last().GetHash(), blocks.Take(5).ToList()).GetAwaiter().GetResult();
// The chain has 10 blocks appended
var chain = new ConcurrentChain(blocks[0].Header);
AppendBlocks(chain, blocks.Skip(1).Take(9));
// Create block store loop
var blockStoreLoop = CreateBlockStoreLoop(chain, blockRepository, @"BlockStore\FindBlocks_CanFind");
// Push blocks 5 - 9 to the downloaded blocks collection
blockStoreLoop.BlockPuller.InjectBlock(blocks[5].GetHash(), new DownloadedBlock()
{
Length = blocks[5].GetSerializedSize(), Block = blocks[5]
}, new CancellationToken());
blockStoreLoop.BlockPuller.InjectBlock(blocks[6].GetHash(), new DownloadedBlock()
{
Length = blocks[6].GetSerializedSize(), Block = blocks[6]
}, new CancellationToken());
blockStoreLoop.BlockPuller.InjectBlock(blocks[7].GetHash(), new DownloadedBlock()
{
Length = blocks[7].GetSerializedSize(), Block = blocks[7]
}, new CancellationToken());
blockStoreLoop.BlockPuller.InjectBlock(blocks[8].GetHash(), new DownloadedBlock()
{
Length = blocks[8].GetSerializedSize(), Block = blocks[8]
}, new CancellationToken());
blockStoreLoop.BlockPuller.InjectBlock(blocks[9].GetHash(), new DownloadedBlock()
{
Length = blocks[9].GetSerializedSize(), Block = blocks[9]
}, new CancellationToken());
//Start processing blocks to download from block 5
var nextChainedBlock = blockStoreLoop.Chain.GetBlock(blocks[5].GetHash());
// Create Task Context
var context = new BlockStoreInnerStepContext(new CancellationToken(), blockStoreLoop).Initialize(nextChainedBlock);
var task = new BlockStoreInnerStepFindBlocks();
task.ExecuteAsync(context).GetAwaiter().GetResult();
//Block[5] and Block[6] in the DownloadStack
Assert.Equal(2, context.DownloadStack.Count());
Assert.True(context.DownloadStack.Any(cb => cb.HashBlock == blocks[5].GetHash()));
Assert.True(context.DownloadStack.Any(cb => cb.HashBlock == blocks[6].GetHash()));
}
}
public void DownloadBlocks_Integration()
{
// Create 10 blocks
var blocks = CreateBlocks(10);
using (var blockRepository = new BlockRepository(Network.Main, TestBase.AssureEmptyDirAsDataFolder(@"BlockStore\DownloadBlocks_Integration")))
{
// Push 5 blocks to the repository
blockRepository.PutAsync(blocks.Take(5).Last().GetHash(), blocks.Take(5).ToList()).GetAwaiter().GetResult();
// The chain has 10 blocks appended
var chain = new ConcurrentChain(blocks[0].Header);
AppendBlocks(chain, blocks.Skip(1).Take(9));
// Create block store loop
var blockStoreLoop = CreateBlockStoreLoop(chain, blockRepository, @"BlockStore\DownloadBlocks_Integration");
// Push blocks 5 - 9 to the downloaded blocks collection
blockStoreLoop.BlockPuller.InjectBlock(blocks[5].GetHash(), new DownloadedBlock()
{
Length = blocks[5].GetSerializedSize(), Block = blocks[5]
}, new CancellationToken());
blockStoreLoop.BlockPuller.InjectBlock(blocks[6].GetHash(), new DownloadedBlock()
{
Length = blocks[6].GetSerializedSize(), Block = blocks[6]
}, new CancellationToken());
blockStoreLoop.BlockPuller.InjectBlock(blocks[7].GetHash(), new DownloadedBlock()
{
Length = blocks[7].GetSerializedSize(), Block = blocks[7]
}, new CancellationToken());
blockStoreLoop.BlockPuller.InjectBlock(blocks[8].GetHash(), new DownloadedBlock()
{
Length = blocks[8].GetSerializedSize(), Block = blocks[8]
}, new CancellationToken());
blockStoreLoop.BlockPuller.InjectBlock(blocks[9].GetHash(), new DownloadedBlock()
{
Length = blocks[9].GetSerializedSize(), Block = blocks[9]
}, new CancellationToken());
// Start processing blocks to download from block 5
var nextChainedBlock = blockStoreLoop.Chain.GetBlock(blocks[5].GetHash());
var step = new DownloadBlockStep(blockStoreLoop);
step.ExecuteAsync(nextChainedBlock, new CancellationToken(), false).GetAwaiter().GetResult();
Assert.Equal(blocks[9].GetHash(), blockStoreLoop.BlockRepository.BlockHash);
Assert.Equal(blocks[9].GetHash(), blockStoreLoop.StoreTip.HashBlock);
}
}
public void BlockRepositoryPutBatch()
{
using (var dir = TestDirectory.Create())
{
using (var blockRepo = new BlockRepository(Network.Main, dir.FolderName, DateTimeProvider.Default, this.loggerFactory))
{
blockRepo.SetTxIndexAsync(true).Wait();
var lst = new List <Block>();
for (int i = 0; i < 5; i++)
{
// put
var block = new Block();
block.AddTransaction(new Transaction());
block.AddTransaction(new Transaction());
block.Transactions[0].AddInput(new TxIn(Script.Empty));
block.Transactions[0].AddOutput(Money.COIN + i * 2, Script.Empty);
block.Transactions[1].AddInput(new TxIn(Script.Empty));
block.Transactions[1].AddOutput(Money.COIN + i * 2 + 1, Script.Empty);
block.UpdateMerkleRoot();
block.Header.HashPrevBlock = lst.Any() ? lst.Last().GetHash() : Network.Main.GenesisHash;
lst.Add(block);
}
blockRepo.PutAsync(lst.Last().GetHash(), lst).GetAwaiter().GetResult();
// check each block
foreach (var block in lst)
{
var received = blockRepo.GetAsync(block.GetHash()).GetAwaiter().GetResult();
Assert.True(block.ToBytes().SequenceEqual(received.ToBytes()));
foreach (var transaction in block.Transactions)
{
var trx = blockRepo.GetTrxAsync(transaction.GetHash()).GetAwaiter().GetResult();
Assert.True(trx.ToBytes().SequenceEqual(transaction.ToBytes()));
}
}
// delete
blockRepo.DeleteAsync(lst.ElementAt(2).GetHash(), new[] { lst.ElementAt(2).GetHash() }.ToList()).GetAwaiter().GetResult();
var deleted = blockRepo.GetAsync(lst.ElementAt(2).GetHash()).GetAwaiter().GetResult();
Assert.Null(deleted);
}
}
}
public void BlockRepositoryPutBatch()
{
using (var blockRepository = new BlockRepository(this.network, TestBase.CreateDataFolder(this), DateTimeProvider.Default, this.loggerFactory))
{
blockRepository.SetTxIndexAsync(true).Wait();
var blocks = new List <Block>();
for (int i = 0; i < 5; i++)
{
Block block = this.network.CreateBlock();
block.AddTransaction(this.network.CreateTransaction());
block.AddTransaction(this.network.CreateTransaction());
block.Transactions[0].AddInput(new TxIn(Script.Empty));
block.Transactions[0].AddOutput(Money.COIN + i * 2, Script.Empty);
block.Transactions[1].AddInput(new TxIn(Script.Empty));
block.Transactions[1].AddOutput(Money.COIN + i * 2 + 1, Script.Empty);
block.UpdateMerkleRoot();
block.Header.HashPrevBlock = blocks.Any() ? blocks.Last().GetHash() : this.network.GenesisHash;
blocks.Add(block);
}
// put
blockRepository.PutAsync(blocks.Last().GetHash(), blocks).GetAwaiter().GetResult();
// check the presence of each block in the repository
foreach (Block block in blocks)
{
Block received = blockRepository.GetAsync(block.GetHash()).GetAwaiter().GetResult();
Assert.True(block.ToBytes().SequenceEqual(received.ToBytes()));
foreach (Transaction transaction in block.Transactions)
{
Transaction trx = blockRepository.GetTrxAsync(transaction.GetHash()).GetAwaiter().GetResult();
Assert.True(trx.ToBytes().SequenceEqual(transaction.ToBytes()));
}
}
// delete
blockRepository.DeleteAsync(blocks.ElementAt(2).GetHash(), new[] { blocks.ElementAt(2).GetHash() }.ToList()).GetAwaiter().GetResult();
Block deleted = blockRepository.GetAsync(blocks.ElementAt(2).GetHash()).GetAwaiter().GetResult();
Assert.Null(deleted);
}
}
public void FindBlocks_CanRemoveTaskFromRoutine_BlockExistsInPendingStorage()
{
// Create 3 blocks
List <Block> blocks = CreateBlocks(3);
// The repository has 3 blocks stored
using (var blockRepository = new BlockRepository(Network.Main, TestBase.AssureEmptyDirAsDataFolder(@"BlockStore\FindBlocks_CanRemoveTaskFromRoutine_BlockExistsInPendingStorage")))
{
blockRepository.PutAsync(blocks.Last().GetHash(), blocks).GetAwaiter().GetResult();
// The chain has 3 blocks appended
var chain = new ConcurrentChain(blocks[0].Header);
AppendBlocks(chain, blocks.Skip(1).Take(2));
// Create block store loop
var blockStoreLoop = CreateBlockStoreLoop(chain, blockRepository, @"BlockStore\FindBlocks_CanRemoveTaskFromRoutine_BlockExistsInPendingStorage");
//Start finding blocks from Block[1]
var nextChainedBlock = blockStoreLoop.Chain.GetBlock(blocks[1].GetHash());
//Add nextChainedBlock to Pending Storage
blockStoreLoop.PendingStorage.TryAdd(blocks[2].GetHash(), new BlockPair(blocks[2], nextChainedBlock));
// Create Task Context
var context = new BlockStoreInnerStepContext(new CancellationToken(), blockStoreLoop).Initialize(nextChainedBlock);
var task = new BlockStoreInnerStepFindBlocks();
task.ExecuteAsync(context).GetAwaiter().GetResult();
//DownloadStack should only contain nextChainedBlock
Assert.Equal(1, context.DownloadStack.Count());
Assert.True(context.DownloadStack.Any(cb => cb.HashBlock == nextChainedBlock.HashBlock));
//The FindBlocks() task should be removed from the routine
//as the next chained block exists in PendingStorage
Assert.Equal(1, context.Routine.Count());
Assert.False(context.Routine.OfType <BlockStoreInnerStepFindBlocks>().Any());
}
}
public void FindBlocks_CanRemoveTaskFromRoutine_BatchDownloadSizeReached()
{
// Create 3 blocks
List <Block> blocks = CreateBlocks(3);
// The repository has 3 blocks stored
using (var blockRepository = new BlockRepository(Network.Main, TestBase.AssureEmptyDirAsDataFolder(@"BlockStore\FindBlocks_CanRemoveTaskFromRoutine_BatchDownloadSizeReached")))
{
blockRepository.PutAsync(blocks.Last().GetHash(), blocks.Take(2).ToList()).GetAwaiter().GetResult();
// The chain has 2 blocks appended
var chain = new ConcurrentChain(blocks[0].Header);
AppendBlocks(chain, blocks.Skip(1).Take(2).ToList());
// Create block store loop
var blockStoreLoop = CreateBlockStoreLoop(chain, blockRepository, @"BlockStore\FindBlocks_CanRemoveTaskFromRoutine_BatchDownloadSizeReached");
blockStoreLoop.BatchDownloadSize = 2;
//Start finding blocks from Block[1]
var nextChainedBlock = blockStoreLoop.Chain.GetBlock(blocks[1].GetHash());
// Create Task Context
var context = new BlockStoreInnerStepContext(new CancellationToken(), blockStoreLoop).Initialize(nextChainedBlock);
var task = new BlockStoreInnerStepFindBlocks();
task.ExecuteAsync(context).GetAwaiter().GetResult();
//Block[1] and Block[2] in the DownloadStack
Assert.Equal(2, context.DownloadStack.Count());
Assert.True(context.DownloadStack.Any(cb => cb.HashBlock == blocks[1].GetHash()));
Assert.True(context.DownloadStack.Any(cb => cb.HashBlock == blocks[2].GetHash()));
//The FindBlocks() task should be removed from the routine
//as the batch download size is reached
Assert.Equal(1, context.Routine.Count());
Assert.False(context.Routine.OfType <BlockStoreInnerStepFindBlocks>().Any());
}
}
public void CanExecute_Reorganise()
{
List <Block> blocks = CreateBlocks(15);
using (var blockRepository = new BlockRepository(Network.Main, TestBase.AssureEmptyDirAsDataFolder(@"BlockStore\CanExecute_Reorganise")))
{
// Push 15 blocks to the repository
blockRepository.PutAsync(blocks.Last().GetHash(), blocks).GetAwaiter().GetResult();
// The chain has 10 blocks appended
var chain = new ConcurrentChain(blocks[0].Header);
AppendBlocks(chain, blocks.Skip(1).Take(9));
// Create the last 5 chained blocks without appending to the chain
var block9 = chain.GetBlock(blocks[9].Header.GetHash());
var block10 = new ChainedBlock(blocks[10].Header, blocks[10].Header.GetHash(), block9);
var block11 = new ChainedBlock(blocks[11].Header, blocks[11].Header.GetHash(), block10);
var block12 = new ChainedBlock(blocks[12].Header, blocks[12].Header.GetHash(), block11);
var block13 = new ChainedBlock(blocks[13].Header, blocks[13].Header.GetHash(), block12);
var block14 = new ChainedBlock(blocks[14].Header, blocks[14].Header.GetHash(), block13);
var blockStoreLoop = CreateBlockStoreLoop(chain, blockRepository, @"BlockStore\CanExecute_Reorganise");
blockStoreLoop.SetStoreTip(block14);
Assert.Equal(blockStoreLoop.StoreTip.Header.GetHash(), block14.Header.GetHash());
Assert.Equal(blockStoreLoop.BlockRepository.BlockHash, block14.Header.GetHash());
//Reorganise (delete) blocks from the block repository that is not found
var nextChainedBlock = block10;
var reorganiseStep = new ReorganiseBlockRepositoryStep(blockStoreLoop);
reorganiseStep.ExecuteAsync(nextChainedBlock, new CancellationToken(), false).GetAwaiter().GetResult();
Assert.Equal(blockStoreLoop.StoreTip.Header.GetHash(), block10.Previous.Header.GetHash());
Assert.Equal(blockStoreLoop.BlockRepository.BlockHash, block10.Previous.Header.GetHash());
}
}