public BlockPuller(OnBlockDownloadedCallback callback, IChainState chainState, ProtocolVersion protocolVersion, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory)
{
this.reassignedJobsQueue = new Queue <DownloadJob>();
this.downloadJobsQueue = new Queue <DownloadJob>();
this.assignedDownloadsByHash = new Dictionary <uint256, AssignedDownload>();
this.assignedDownloadsSorted = new LinkedList <AssignedDownload>();
this.assignedHeadersByPeerId = new Dictionary <int, List <ChainedHeader> >();
this.averageBlockSizeBytes = new AverageCalculator(AverageBlockSizeSamplesCount);
this.pullerBehaviorsByPeerId = new Dictionary <int, IBlockPullerBehavior>();
this.processQueuesSignal = new AsyncManualResetEvent(false);
this.queueLock = new object();
this.peerLock = new object();
this.assignedLock = new object();
this.nextJobId = 0;
this.networkPeerRequirement = new NetworkPeerRequirement
{
MinVersion = protocolVersion,
RequiredServices = NetworkPeerServices.Network
};
this.cancellationSource = new CancellationTokenSource();
this.random = new Random();
this.maxBlocksBeingDownloaded = MinimalCountOfBlocksBeingDownloaded;
this.onDownloadedCallback = callback;
this.chainState = chainState;
this.dateTimeProvider = dateTimeProvider;
this.logger = loggerFactory.CreateLogger(this.GetType().FullName);
}
/// <inheritdoc/>
public void Initialize(OnBlockDownloadedCallback callback)
{
this.onDownloadedCallback = callback;
this.assignerLoop = this.AssignerLoopAsync();
this.stallingLoop = this.StallingLoopAsync();
}
public void BlockDownloaded_ExpectedBlockDataBytesCalculatedCorrectly()
{
TestContext builder = GetBuildTestContext(10, useCheckpoints: false, initializeWithChainTip: true);
builder.SetupAverageBlockSize(100);
var additionalHeaders = builder.ExtendAChain(1, builder.InitialChainTip, avgBlockSize: 100);
var headerTree = builder.ChainedHeaderToList(additionalHeaders, 1);
var peer = builder.GetNetworkPeerWithConnection();
var result = builder.TestConsensusManager.ConsensusManager.HeadersPresented(peer.Object, headerTree, true);
Assert.NotNull(builder.blockPullerBlockDownloadCallback);
builder.TestConsensusManager.SetExpectedBlockDataBytes(1000);
var callback1Called = false;
var callback2Called = false;
var callback1 = new OnBlockDownloadedCallback(d => { callback1Called = true; });
var callback2 = new OnBlockDownloadedCallback(d => { callback2Called = true; });
builder.TestConsensusManager.SetupCallbackByBlocksRequestedHash(additionalHeaders.HashBlock, callback1, callback2);
builder.blockPullerBlockDownloadCallback(additionalHeaders.HashBlock, additionalHeaders.Block, peer.Object.Connection.Id);
Assert.False(builder.TestConsensusManager.CallbacksByBlocksRequestedHashContainsKeyForHash(additionalHeaders.HashBlock));
Assert.Equal(900, builder.TestConsensusManager.GetExpectedBlockDataBytes());
builder.AssertExpectedBlockSizesEmpty();
Assert.True(callback1Called);
Assert.True(callback2Called);
}
public void BlockDownloaded_KnownHeader_BlockIntegrityInvalidated_BansPeer_DoesNotCallCallback()
{
TestContext builder = GetBuildTestContext(10, useCheckpoints: false, initializeWithChainTip: true);
var additionalHeaders = builder.ExtendAChain(1, builder.InitialChainTip);
var headerTree = builder.ChainedHeaderToList(additionalHeaders, 1);
var peer = builder.GetNetworkPeerWithConnection();
var result = builder.TestConsensusManager.ConsensusManager.HeadersPresented(peer.Object, headerTree, true);
var callbackCalled = false;
var callback = new OnBlockDownloadedCallback(d => { callbackCalled = true; });
// setup the callback
builder.TestConsensusManager.SetupCallbackByBlocksRequestedHash(additionalHeaders.HashBlock, callback);
Assert.NotNull(builder.blockPullerBlockDownloadCallback);
// setup validation to fail.
builder.IntegrityValidator.Setup(i => i.VerifyBlockIntegrity(additionalHeaders, additionalHeaders.Block))
.Returns(new ValidationContext()
{
BanDurationSeconds = 3000,
Error = ConsensusErrors.BadBlockSignature
});
builder.blockPullerBlockDownloadCallback(additionalHeaders.HashBlock, additionalHeaders.Block, peer.Object.Connection.Id);
// expect the setup callback is not called.
Assert.False(callbackCalled);
builder.AssertPeerBanned(peer.Object);
builder.AssertExpectedBlockSizes(builder.Network.Consensus.Options.MaxBlockBaseSize);
}
public void BlockDownloaded_CallbackRegisteredForHash_KnownHeader_NotBlockExpected_ThrowsInvalidOperationException()
{
TestContext builder = GetBuildTestContext(10, useCheckpoints: false, initializeWithChainTip: true);
var additionalHeaders = builder.ExtendAChain(1, builder.InitialChainTip);
var headerTree = builder.ChainedHeaderToList(additionalHeaders, 1);
var peer = builder.GetNetworkPeerWithConnection();
var result = builder.TestConsensusManager.ConsensusManager.HeadersPresented(peer.Object, headerTree, true);
builder.TestConsensusManager.ClearExpectedBlockSizes();
var callbackCalled = false;
var callback = new OnBlockDownloadedCallback(d => { callbackCalled = true; });
// setup the callback
builder.TestConsensusManager.SetupCallbackByBlocksRequestedHash(additionalHeaders.HashBlock, callback);
Assert.NotNull(builder.blockPullerBlockDownloadCallback);
// call the blockdownloaded method from the blockpuller with a random header
Assert.Throws <InvalidOperationException>(() => builder.blockPullerBlockDownloadCallback(additionalHeaders.HashBlock, null, peer.Object.Connection.Id));
// expect the setup callback is not called.
Assert.False(callbackCalled);
}
/// <summary>
/// Provides block data for the given block hashes.
/// </summary>
/// <remarks>
/// First we check if the block exists in chained header tree, then it check the block store and if it wasn't found there the block will be scheduled for download.
/// Given callback is called when the block is obtained. If obtaining the block fails the callback will be called with <c>null</c>.
/// </remarks>
/// <param name="blockHashes">The block hashes to download.</param>
/// <param name="onBlockDownloadedCallback">The callback that will be called for each downloaded block.</param>
public async Task GetOrDownloadBlocksAsync(List <uint256> blockHashes, OnBlockDownloadedCallback onBlockDownloadedCallback)
{
this.logger.LogTrace("({0}.{1}:{2})", nameof(blockHashes), nameof(blockHashes.Count), blockHashes.Count);
var blocksToDownload = new List <ChainedHeader>();
for (int i = 0; i < blockHashes.Count; i++)
{
uint256 blockHash = blockHashes[i];
ChainedHeaderBlock chainedHeaderBlock = null;
lock (this.peerLock)
{
chainedHeaderBlock = this.chainedHeaderTree.GetChainedHeaderBlock(blockHash);
}
if (chainedHeaderBlock == null)
{
this.logger.LogTrace("Block hash '{0}' is not part of the tree.", blockHash);
onBlockDownloadedCallback(null);
continue;
}
if (chainedHeaderBlock.Block != null)
{
this.logger.LogTrace("Block pair '{0}' was found in memory.", chainedHeaderBlock);
onBlockDownloadedCallback(chainedHeaderBlock);
continue;
}
if (this.blockStore != null)
{
Block block = await this.blockStore.GetBlockAsync(blockHash).ConfigureAwait(false);
if (block != null)
{
var newBlockPair = new ChainedHeaderBlock(block, chainedHeaderBlock.ChainedHeader);
this.logger.LogTrace("Chained header block '{0}' was found in store.", newBlockPair);
onBlockDownloadedCallback(newBlockPair);
continue;
}
}
blocksToDownload.Add(chainedHeaderBlock.ChainedHeader);
this.logger.LogTrace("Block hash '{0}' is queued for download.", blockHash);
}
if (blocksToDownload.Count != 0)
{
this.logger.LogTrace("Asking block puller for {0} blocks.", blocksToDownload.Count);
this.DownloadBlocks(blocksToDownload.ToArray(), this.ProcessDownloadedBlock);
}
this.logger.LogTrace("(-)");
}
public void BlockDownloaded_NullBlock_CallsCallbacks()
{
TestContext builder = GetBuildTestContext(10, useCheckpoints: false, initializeWithChainTip: true);
var additionalHeaders = builder.ExtendAChain(2, builder.InitialChainTip);
var headerTree = builder.ChainedHeaderToList(additionalHeaders, 2);
var peer = builder.GetNetworkPeerWithConnection();
var result = builder.TestConsensusManager.ConsensusManager.HeadersPresented(peer.Object, headerTree, true);
Assert.NotNull(builder.blockPullerBlockDownloadCallback);
builder.TestConsensusManager.SetExpectedBlockDataBytes(1000);
var callback1Called = false;
var callback2Called = false;
var callback3Called = false;
ChainedHeaderBlock calledWith1 = null;
ChainedHeaderBlock calledWith2 = null;
ChainedHeaderBlock calledWith3 = null;
var callback1 = new OnBlockDownloadedCallback(d => { callback1Called = true; calledWith1 = d; });
var callback2 = new OnBlockDownloadedCallback(d => { callback2Called = true; calledWith2 = d; });
var callback3 = new OnBlockDownloadedCallback(d => { callback3Called = true; calledWith3 = d; });
builder.TestConsensusManager.SetupCallbackByBlocksRequestedHash(additionalHeaders.HashBlock, callback1, callback2);
builder.TestConsensusManager.SetupCallbackByBlocksRequestedHash(additionalHeaders.Previous.HashBlock, callback3);
// call for both blocks.
var block = new Block();
block.ToBytes();
builder.blockPullerBlockDownloadCallback(additionalHeaders.Previous.HashBlock, block, peer.Object.Connection.Id);
builder.blockPullerBlockDownloadCallback(additionalHeaders.HashBlock, block, peer.Object.Connection.Id);
Assert.False(builder.TestConsensusManager.CallbacksByBlocksRequestedHashContainsKeyForHash(additionalHeaders.HashBlock));
Assert.False(builder.TestConsensusManager.CallbacksByBlocksRequestedHashContainsKeyForHash(additionalHeaders.Previous.HashBlock));
builder.AssertExpectedBlockSizesEmpty();
Assert.True(callback1Called);
Assert.True(callback2Called);
Assert.True(callback3Called);
Assert.NotNull(calledWith1);
Assert.Equal(additionalHeaders, calledWith1.ChainedHeader);
Assert.NotNull(calledWith2);
Assert.Equal(additionalHeaders, calledWith2.ChainedHeader);
Assert.NotNull(calledWith3);
Assert.Equal(additionalHeaders.Previous, calledWith3.ChainedHeader);
}
public void GetOrDownloadBlocksAsync_ChainedHeaderBlockNotInCT_CallsBlockDownloadedCallbackForBlock_BlocksNotDownloaded()
{
TestContext builder = GetBuildTestContext(10, useCheckpoints: false);
var callbackCalled = false;
ChainedHeaderBlock calledWith = null;
var blockDownloadedCallback = new OnBlockDownloadedCallback(d => { callbackCalled = true; calledWith = d; });
var blockHashes = new List <uint256>()
{
builder.InitialChainTip.HashBlock
};
builder.TestConsensusManager.ConsensusManager.GetOrDownloadBlocks(blockHashes, blockDownloadedCallback);
Assert.True(callbackCalled);
Assert.Null(calledWith);
builder.BlockPuller.Verify(b => b.RequestBlocksDownload(It.IsAny <List <ChainedHeader> >(), It.IsAny <bool>()), Times.Exactly(0));
}
public void GetOrDownloadBlocksAsync_ChainedHeaderBlockInCTWithoutBlock_DoesNotCallBlockDownloadedCallbackForBlock_BlockDownloaded()
{
TestContext builder = GetBuildTestContext(10, useCheckpoints: false);
var blockHashes = new List <uint256>()
{
builder.InitialChainTip.HashBlock
};
builder.InitialChainTip.Block = null;
builder.TestConsensusManager.ConsensusManager.InitializeAsync(builder.InitialChainTip).GetAwaiter().GetResult();
var callbackCalled = false;
ChainedHeaderBlock calledWith = null;
var blockDownloadedCallback = new OnBlockDownloadedCallback(d => { callbackCalled = true; calledWith = d; });
builder.TestConsensusManager.ConsensusManager.GetOrDownloadBlocks(blockHashes, blockDownloadedCallback);
Assert.True(builder.TestConsensusManager.CallbacksByBlocksRequestedHashContainsKeyForHash(builder.InitialChainTip.HashBlock));
Assert.False(callbackCalled);
builder.BlockPuller.Verify(b => b.RequestBlocksDownload(It.IsAny <List <ChainedHeader> >(), It.IsAny <bool>()), Times.Exactly(1));
}
/// <inheritdoc />
public async Task GetOrDownloadBlocksAsync(List <uint256> blockHashes, OnBlockDownloadedCallback onBlockDownloadedCallback)
{
this.logger.LogTrace("({0}.{1}:{2})", nameof(blockHashes), nameof(blockHashes.Count), blockHashes.Count);
var blocksToDownload = new List <ChainedHeader>();
foreach (uint256 blockHash in blockHashes)
{
ChainedHeaderBlock chainedHeaderBlock = await this.LoadBlockDataAsync(blockHash).ConfigureAwait(false);
if ((chainedHeaderBlock == null) || (chainedHeaderBlock.Block != null))
{
if (chainedHeaderBlock != null)
{
this.logger.LogTrace("Block data loaded for hash '{0}', calling the callback.", blockHash);
}
else
{
this.logger.LogTrace("Chained header not found for hash '{0}'.", blockHash);
}
onBlockDownloadedCallback(chainedHeaderBlock);
}
else
{
blocksToDownload.Add(chainedHeaderBlock.ChainedHeader);
this.logger.LogTrace("Block hash '{0}' is queued for download.", blockHash);
}
}
if (blocksToDownload.Count != 0)
{
this.logger.LogTrace("Asking block puller for {0} blocks.", blocksToDownload.Count);
this.DownloadBlocks(blocksToDownload.ToArray(), this.ProcessDownloadedBlock);
}
this.logger.LogTrace("(-)");
}
/// <summary>
/// Request a list of block headers to download their respective blocks.
/// If <paramref name="chainedHeaders"/> is not an array of consecutive headers it will be split to batches of consecutive header requests.
/// Callbacks of all entries are added to <see cref="callbacksByBlocksRequestedHash"/>. If a block header was already requested
/// to download and not delivered yet, it will not be requested again, instead just it's callback will be called when the block arrives.
/// </summary>
/// <param name="chainedHeaders">Array of chained headers to download.</param>
/// <param name="onBlockDownloadedCallback">A callback to call when the block was downloaded.</param>
private void DownloadBlocks(ChainedHeader[] chainedHeaders, OnBlockDownloadedCallback onBlockDownloadedCallback)
{
this.logger.LogTrace("({0}.{1}:{2})", nameof(chainedHeaders), nameof(chainedHeaders.Length), chainedHeaders.Length);
var downloadRequests = new List <BlockDownloadRequest>();
BlockDownloadRequest request = null;
ChainedHeader previousHeader = null;
lock (this.blockRequestedLock)
{
foreach (ChainedHeader chainedHeader in chainedHeaders)
{
bool blockAlreadyAsked = this.callbacksByBlocksRequestedHash.TryGetValue(chainedHeader.HashBlock, out List <OnBlockDownloadedCallback> callbacks);
if (!blockAlreadyAsked)
{
callbacks = new List <OnBlockDownloadedCallback>();
this.callbacksByBlocksRequestedHash.Add(chainedHeader.HashBlock, callbacks);
}
else
{
this.logger.LogTrace("Registered additional callback for the block '{0}'.", chainedHeader);
}
callbacks.Add(onBlockDownloadedCallback);
bool blockIsNotConsecutive = (previousHeader != null) && (chainedHeader.Previous.HashBlock != previousHeader.HashBlock);
if (blockIsNotConsecutive || blockAlreadyAsked)
{
if (request != null)
{
downloadRequests.Add(request);
request = null;
}
if (blockAlreadyAsked)
{
previousHeader = null;
continue;
}
}
if (request == null)
{
request = new BlockDownloadRequest {
BlocksToDownload = new List <ChainedHeader>()
}
}
;
request.BlocksToDownload.Add(chainedHeader);
previousHeader = chainedHeader;
}
if (request != null)
{
downloadRequests.Add(request);
}
lock (this.peerLock)
{
foreach (BlockDownloadRequest downloadRequest in downloadRequests)
{
this.toDownloadQueue.Enqueue(downloadRequest);
}
this.ProcessDownloadQueueLocked();
}
}
this.logger.LogTrace("(-)");
}
