public BlockPuller(OnBlockDownloadedCallback callback, IChainState chainState, ProtocolVersion protocolVersion, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory) { this.reassignedJobsQueue = new Queue <DownloadJob>(); this.downloadJobsQueue = new Queue <DownloadJob>(); this.assignedDownloadsByHash = new Dictionary <uint256, AssignedDownload>(); this.assignedDownloadsSorted = new LinkedList <AssignedDownload>(); this.assignedHeadersByPeerId = new Dictionary <int, List <ChainedHeader> >(); this.averageBlockSizeBytes = new AverageCalculator(AverageBlockSizeSamplesCount); this.pullerBehaviorsByPeerId = new Dictionary <int, IBlockPullerBehavior>(); this.processQueuesSignal = new AsyncManualResetEvent(false); this.queueLock = new object(); this.peerLock = new object(); this.assignedLock = new object(); this.nextJobId = 0; this.networkPeerRequirement = new NetworkPeerRequirement { MinVersion = protocolVersion, RequiredServices = NetworkPeerServices.Network }; this.cancellationSource = new CancellationTokenSource(); this.random = new Random(); this.maxBlocksBeingDownloaded = MinimalCountOfBlocksBeingDownloaded; this.onDownloadedCallback = callback; this.chainState = chainState; this.dateTimeProvider = dateTimeProvider; this.logger = loggerFactory.CreateLogger(this.GetType().FullName); }
/// <inheritdoc/> public void Initialize(OnBlockDownloadedCallback callback) { this.onDownloadedCallback = callback; this.assignerLoop = this.AssignerLoopAsync(); this.stallingLoop = this.StallingLoopAsync(); }
public void BlockDownloaded_ExpectedBlockDataBytesCalculatedCorrectly() { TestContext builder = GetBuildTestContext(10, useCheckpoints: false, initializeWithChainTip: true); builder.SetupAverageBlockSize(100); var additionalHeaders = builder.ExtendAChain(1, builder.InitialChainTip, avgBlockSize: 100); var headerTree = builder.ChainedHeaderToList(additionalHeaders, 1); var peer = builder.GetNetworkPeerWithConnection(); var result = builder.TestConsensusManager.ConsensusManager.HeadersPresented(peer.Object, headerTree, true); Assert.NotNull(builder.blockPullerBlockDownloadCallback); builder.TestConsensusManager.SetExpectedBlockDataBytes(1000); var callback1Called = false; var callback2Called = false; var callback1 = new OnBlockDownloadedCallback(d => { callback1Called = true; }); var callback2 = new OnBlockDownloadedCallback(d => { callback2Called = true; }); builder.TestConsensusManager.SetupCallbackByBlocksRequestedHash(additionalHeaders.HashBlock, callback1, callback2); builder.blockPullerBlockDownloadCallback(additionalHeaders.HashBlock, additionalHeaders.Block, peer.Object.Connection.Id); Assert.False(builder.TestConsensusManager.CallbacksByBlocksRequestedHashContainsKeyForHash(additionalHeaders.HashBlock)); Assert.Equal(900, builder.TestConsensusManager.GetExpectedBlockDataBytes()); builder.AssertExpectedBlockSizesEmpty(); Assert.True(callback1Called); Assert.True(callback2Called); }
public void BlockDownloaded_KnownHeader_BlockIntegrityInvalidated_BansPeer_DoesNotCallCallback() { TestContext builder = GetBuildTestContext(10, useCheckpoints: false, initializeWithChainTip: true); var additionalHeaders = builder.ExtendAChain(1, builder.InitialChainTip); var headerTree = builder.ChainedHeaderToList(additionalHeaders, 1); var peer = builder.GetNetworkPeerWithConnection(); var result = builder.TestConsensusManager.ConsensusManager.HeadersPresented(peer.Object, headerTree, true); var callbackCalled = false; var callback = new OnBlockDownloadedCallback(d => { callbackCalled = true; }); // setup the callback builder.TestConsensusManager.SetupCallbackByBlocksRequestedHash(additionalHeaders.HashBlock, callback); Assert.NotNull(builder.blockPullerBlockDownloadCallback); // setup validation to fail. builder.IntegrityValidator.Setup(i => i.VerifyBlockIntegrity(additionalHeaders, additionalHeaders.Block)) .Returns(new ValidationContext() { BanDurationSeconds = 3000, Error = ConsensusErrors.BadBlockSignature }); builder.blockPullerBlockDownloadCallback(additionalHeaders.HashBlock, additionalHeaders.Block, peer.Object.Connection.Id); // expect the setup callback is not called. Assert.False(callbackCalled); builder.AssertPeerBanned(peer.Object); builder.AssertExpectedBlockSizes(builder.Network.Consensus.Options.MaxBlockBaseSize); }
public void BlockDownloaded_CallbackRegisteredForHash_KnownHeader_NotBlockExpected_ThrowsInvalidOperationException() { TestContext builder = GetBuildTestContext(10, useCheckpoints: false, initializeWithChainTip: true); var additionalHeaders = builder.ExtendAChain(1, builder.InitialChainTip); var headerTree = builder.ChainedHeaderToList(additionalHeaders, 1); var peer = builder.GetNetworkPeerWithConnection(); var result = builder.TestConsensusManager.ConsensusManager.HeadersPresented(peer.Object, headerTree, true); builder.TestConsensusManager.ClearExpectedBlockSizes(); var callbackCalled = false; var callback = new OnBlockDownloadedCallback(d => { callbackCalled = true; }); // setup the callback builder.TestConsensusManager.SetupCallbackByBlocksRequestedHash(additionalHeaders.HashBlock, callback); Assert.NotNull(builder.blockPullerBlockDownloadCallback); // call the blockdownloaded method from the blockpuller with a random header Assert.Throws <InvalidOperationException>(() => builder.blockPullerBlockDownloadCallback(additionalHeaders.HashBlock, null, peer.Object.Connection.Id)); // expect the setup callback is not called. Assert.False(callbackCalled); }
/// <summary> /// Provides block data for the given block hashes. /// </summary> /// <remarks> /// First we check if the block exists in chained header tree, then it check the block store and if it wasn't found there the block will be scheduled for download. /// Given callback is called when the block is obtained. If obtaining the block fails the callback will be called with <c>null</c>. /// </remarks> /// <param name="blockHashes">The block hashes to download.</param> /// <param name="onBlockDownloadedCallback">The callback that will be called for each downloaded block.</param> public async Task GetOrDownloadBlocksAsync(List <uint256> blockHashes, OnBlockDownloadedCallback onBlockDownloadedCallback) { this.logger.LogTrace("({0}.{1}:{2})", nameof(blockHashes), nameof(blockHashes.Count), blockHashes.Count); var blocksToDownload = new List <ChainedHeader>(); for (int i = 0; i < blockHashes.Count; i++) { uint256 blockHash = blockHashes[i]; ChainedHeaderBlock chainedHeaderBlock = null; lock (this.peerLock) { chainedHeaderBlock = this.chainedHeaderTree.GetChainedHeaderBlock(blockHash); } if (chainedHeaderBlock == null) { this.logger.LogTrace("Block hash '{0}' is not part of the tree.", blockHash); onBlockDownloadedCallback(null); continue; } if (chainedHeaderBlock.Block != null) { this.logger.LogTrace("Block pair '{0}' was found in memory.", chainedHeaderBlock); onBlockDownloadedCallback(chainedHeaderBlock); continue; } if (this.blockStore != null) { Block block = await this.blockStore.GetBlockAsync(blockHash).ConfigureAwait(false); if (block != null) { var newBlockPair = new ChainedHeaderBlock(block, chainedHeaderBlock.ChainedHeader); this.logger.LogTrace("Chained header block '{0}' was found in store.", newBlockPair); onBlockDownloadedCallback(newBlockPair); continue; } } blocksToDownload.Add(chainedHeaderBlock.ChainedHeader); this.logger.LogTrace("Block hash '{0}' is queued for download.", blockHash); } if (blocksToDownload.Count != 0) { this.logger.LogTrace("Asking block puller for {0} blocks.", blocksToDownload.Count); this.DownloadBlocks(blocksToDownload.ToArray(), this.ProcessDownloadedBlock); } this.logger.LogTrace("(-)"); }
public void BlockDownloaded_NullBlock_CallsCallbacks() { TestContext builder = GetBuildTestContext(10, useCheckpoints: false, initializeWithChainTip: true); var additionalHeaders = builder.ExtendAChain(2, builder.InitialChainTip); var headerTree = builder.ChainedHeaderToList(additionalHeaders, 2); var peer = builder.GetNetworkPeerWithConnection(); var result = builder.TestConsensusManager.ConsensusManager.HeadersPresented(peer.Object, headerTree, true); Assert.NotNull(builder.blockPullerBlockDownloadCallback); builder.TestConsensusManager.SetExpectedBlockDataBytes(1000); var callback1Called = false; var callback2Called = false; var callback3Called = false; ChainedHeaderBlock calledWith1 = null; ChainedHeaderBlock calledWith2 = null; ChainedHeaderBlock calledWith3 = null; var callback1 = new OnBlockDownloadedCallback(d => { callback1Called = true; calledWith1 = d; }); var callback2 = new OnBlockDownloadedCallback(d => { callback2Called = true; calledWith2 = d; }); var callback3 = new OnBlockDownloadedCallback(d => { callback3Called = true; calledWith3 = d; }); builder.TestConsensusManager.SetupCallbackByBlocksRequestedHash(additionalHeaders.HashBlock, callback1, callback2); builder.TestConsensusManager.SetupCallbackByBlocksRequestedHash(additionalHeaders.Previous.HashBlock, callback3); // call for both blocks. var block = new Block(); block.ToBytes(); builder.blockPullerBlockDownloadCallback(additionalHeaders.Previous.HashBlock, block, peer.Object.Connection.Id); builder.blockPullerBlockDownloadCallback(additionalHeaders.HashBlock, block, peer.Object.Connection.Id); Assert.False(builder.TestConsensusManager.CallbacksByBlocksRequestedHashContainsKeyForHash(additionalHeaders.HashBlock)); Assert.False(builder.TestConsensusManager.CallbacksByBlocksRequestedHashContainsKeyForHash(additionalHeaders.Previous.HashBlock)); builder.AssertExpectedBlockSizesEmpty(); Assert.True(callback1Called); Assert.True(callback2Called); Assert.True(callback3Called); Assert.NotNull(calledWith1); Assert.Equal(additionalHeaders, calledWith1.ChainedHeader); Assert.NotNull(calledWith2); Assert.Equal(additionalHeaders, calledWith2.ChainedHeader); Assert.NotNull(calledWith3); Assert.Equal(additionalHeaders.Previous, calledWith3.ChainedHeader); }
public void GetOrDownloadBlocksAsync_ChainedHeaderBlockNotInCT_CallsBlockDownloadedCallbackForBlock_BlocksNotDownloaded() { TestContext builder = GetBuildTestContext(10, useCheckpoints: false); var callbackCalled = false; ChainedHeaderBlock calledWith = null; var blockDownloadedCallback = new OnBlockDownloadedCallback(d => { callbackCalled = true; calledWith = d; }); var blockHashes = new List <uint256>() { builder.InitialChainTip.HashBlock }; builder.TestConsensusManager.ConsensusManager.GetOrDownloadBlocks(blockHashes, blockDownloadedCallback); Assert.True(callbackCalled); Assert.Null(calledWith); builder.BlockPuller.Verify(b => b.RequestBlocksDownload(It.IsAny <List <ChainedHeader> >(), It.IsAny <bool>()), Times.Exactly(0)); }
public void GetOrDownloadBlocksAsync_ChainedHeaderBlockInCTWithoutBlock_DoesNotCallBlockDownloadedCallbackForBlock_BlockDownloaded() { TestContext builder = GetBuildTestContext(10, useCheckpoints: false); var blockHashes = new List <uint256>() { builder.InitialChainTip.HashBlock }; builder.InitialChainTip.Block = null; builder.TestConsensusManager.ConsensusManager.InitializeAsync(builder.InitialChainTip).GetAwaiter().GetResult(); var callbackCalled = false; ChainedHeaderBlock calledWith = null; var blockDownloadedCallback = new OnBlockDownloadedCallback(d => { callbackCalled = true; calledWith = d; }); builder.TestConsensusManager.ConsensusManager.GetOrDownloadBlocks(blockHashes, blockDownloadedCallback); Assert.True(builder.TestConsensusManager.CallbacksByBlocksRequestedHashContainsKeyForHash(builder.InitialChainTip.HashBlock)); Assert.False(callbackCalled); builder.BlockPuller.Verify(b => b.RequestBlocksDownload(It.IsAny <List <ChainedHeader> >(), It.IsAny <bool>()), Times.Exactly(1)); }
/// <inheritdoc /> public async Task GetOrDownloadBlocksAsync(List <uint256> blockHashes, OnBlockDownloadedCallback onBlockDownloadedCallback) { this.logger.LogTrace("({0}.{1}:{2})", nameof(blockHashes), nameof(blockHashes.Count), blockHashes.Count); var blocksToDownload = new List <ChainedHeader>(); foreach (uint256 blockHash in blockHashes) { ChainedHeaderBlock chainedHeaderBlock = await this.LoadBlockDataAsync(blockHash).ConfigureAwait(false); if ((chainedHeaderBlock == null) || (chainedHeaderBlock.Block != null)) { if (chainedHeaderBlock != null) { this.logger.LogTrace("Block data loaded for hash '{0}', calling the callback.", blockHash); } else { this.logger.LogTrace("Chained header not found for hash '{0}'.", blockHash); } onBlockDownloadedCallback(chainedHeaderBlock); } else { blocksToDownload.Add(chainedHeaderBlock.ChainedHeader); this.logger.LogTrace("Block hash '{0}' is queued for download.", blockHash); } } if (blocksToDownload.Count != 0) { this.logger.LogTrace("Asking block puller for {0} blocks.", blocksToDownload.Count); this.DownloadBlocks(blocksToDownload.ToArray(), this.ProcessDownloadedBlock); } this.logger.LogTrace("(-)"); }
/// <summary> /// Request a list of block headers to download their respective blocks. /// If <paramref name="chainedHeaders"/> is not an array of consecutive headers it will be split to batches of consecutive header requests. /// Callbacks of all entries are added to <see cref="callbacksByBlocksRequestedHash"/>. If a block header was already requested /// to download and not delivered yet, it will not be requested again, instead just it's callback will be called when the block arrives. /// </summary> /// <param name="chainedHeaders">Array of chained headers to download.</param> /// <param name="onBlockDownloadedCallback">A callback to call when the block was downloaded.</param> private void DownloadBlocks(ChainedHeader[] chainedHeaders, OnBlockDownloadedCallback onBlockDownloadedCallback) { this.logger.LogTrace("({0}.{1}:{2})", nameof(chainedHeaders), nameof(chainedHeaders.Length), chainedHeaders.Length); var downloadRequests = new List <BlockDownloadRequest>(); BlockDownloadRequest request = null; ChainedHeader previousHeader = null; lock (this.blockRequestedLock) { foreach (ChainedHeader chainedHeader in chainedHeaders) { bool blockAlreadyAsked = this.callbacksByBlocksRequestedHash.TryGetValue(chainedHeader.HashBlock, out List <OnBlockDownloadedCallback> callbacks); if (!blockAlreadyAsked) { callbacks = new List <OnBlockDownloadedCallback>(); this.callbacksByBlocksRequestedHash.Add(chainedHeader.HashBlock, callbacks); } else { this.logger.LogTrace("Registered additional callback for the block '{0}'.", chainedHeader); } callbacks.Add(onBlockDownloadedCallback); bool blockIsNotConsecutive = (previousHeader != null) && (chainedHeader.Previous.HashBlock != previousHeader.HashBlock); if (blockIsNotConsecutive || blockAlreadyAsked) { if (request != null) { downloadRequests.Add(request); request = null; } if (blockAlreadyAsked) { previousHeader = null; continue; } } if (request == null) { request = new BlockDownloadRequest { BlocksToDownload = new List <ChainedHeader>() } } ; request.BlocksToDownload.Add(chainedHeader); previousHeader = chainedHeader; } if (request != null) { downloadRequests.Add(request); } lock (this.peerLock) { foreach (BlockDownloadRequest downloadRequest in downloadRequests) { this.toDownloadQueue.Enqueue(downloadRequest); } this.ProcessDownloadQueueLocked(); } } this.logger.LogTrace("(-)"); }