public async Task Can_sync_with_peer_when_it_times_out_on_full_batch()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(async ci => await _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect | Response.TimeoutOnFullBatch));
syncPeer.GetBlocks(Arg.Any <Keccak[]>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <Keccak[]>(0), Response.AllCorrect | Response.TimeoutOnFullBatch));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = SyncBatchSize.Max * 2 + 32;
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None).ContinueWith(t => { });
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
Assert.AreEqual(Math.Max(0, peerInfo.HeadNumber - SyncModeSelector.FullSyncThreshold), _blockTree.BestSuggested.Number);
peerInfo.HeadNumber *= 2;
await blockDownloader.DownloadBlocks(peerInfo, CancellationToken.None).ContinueWith(t => { });
await blockDownloader.DownloadBlocks(peerInfo, CancellationToken.None);
Assert.AreEqual(Math.Max(0, peerInfo.HeadNumber), _blockTree.BestSuggested.Number);
}
public async Task Can_cancel_adding_headers()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, new SlowHeaderValidator(), TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
syncPeer.GetBlocks(Arg.Any <Keccak[]>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <Keccak[]>(0), Response.AllCorrect));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = 1000;
CancellationTokenSource cancellation = new CancellationTokenSource();
cancellation.CancelAfter(1000);
Task task = blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, cancellation.Token);
await task.ContinueWith(t => Assert.True(t.IsCanceled, "headers"));
peerInfo.HeadNumber *= 2;
cancellation = new CancellationTokenSource();
cancellation.CancelAfter(1000);
task = blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, cancellation.Token);
await task.ContinueWith(t => Assert.True(t.IsCanceled, "blocks"));
}
public async Task Throws_on_incorrect_receipts_root()
{
InMemoryReceiptStorage inMemoryReceiptStorage = new InMemoryReceiptStorage();
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, NullSyncReport.Instance, inMemoryReceiptStorage, RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
Task <BlockHeader[]> buildHeadersResponse = null;
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => buildHeadersResponse = _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
Task <BlockBody[]> buildBlocksResponse = null;
syncPeer.GetBlocks(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => buildBlocksResponse = _responseBuilder.BuildBlocksResponse(ci.ArgAt <IList <Keccak> >(0), Response.AllCorrect | Response.WithTransactions));
syncPeer.GetReceipts(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildReceiptsResponse(buildHeadersResponse.Result, buildBlocksResponse.Result, Response.IncorrectReceiptRoot).Result);
PeerInfo peerInfo = new PeerInfo(syncPeer)
{
TotalDifficulty = UInt256.MaxValue, HeadNumber = 1
};
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
peerInfo.HeadNumber *= 2;
Func <Task> action = async() => await blockDownloader.DownloadBlocks(peerInfo, 0, CancellationToken.None, BlockDownloader.DownloadOptions.DownloadWithReceipts);
action.Should().Throw <EthSynchronizationException>();
}
public async Task Happy_path(long headNumber)
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
syncPeer.GetBlocks(Arg.Any <Keccak[]>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <Keccak[]>(0), Response.AllCorrect));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = headNumber;
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
Assert.AreEqual(Math.Max(0, headNumber - SyncModeSelector.FullSyncThreshold), _blockTree.BestSuggested.Number, "headers");
peerInfo.HeadNumber *= 2;
await blockDownloader.DownloadBlocks(peerInfo, CancellationToken.None);
Assert.AreEqual(Math.Max(0, headNumber * 2), _blockTree.BestSuggested.Number);
}
public async Task Can_cancel_seal_validation()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, new SlowSealValidator(), NullSyncReport.Instance, new InMemoryReceiptStorage(), RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
syncPeer.GetBlocks(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <IList <Keccak> >(0), Response.AllCorrect));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = 1000;
CancellationTokenSource cancellation = new CancellationTokenSource();
cancellation.CancelAfter(1000);
Task task = blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, cancellation.Token);
await task.ContinueWith(t => Assert.True(t.IsCanceled, $"headers {t.Status}"));
peerInfo.HeadNumber = 2000;
cancellation = new CancellationTokenSource();
cancellation.CancelAfter(1000);
task = blockDownloader.DownloadBlocks(peerInfo, 0, cancellation.Token);
await task.ContinueWith(t => Assert.True(t.IsCanceled, $"blocks {t.Status}"));
}
public async Task Happy_path(long headNumber, int options)
{
BlockDownloader.DownloadOptions downloadOptions = (BlockDownloader.DownloadOptions)options;
bool withReceipts = downloadOptions == BlockDownloader.DownloadOptions.DownloadWithReceipts;
InMemoryReceiptStorage inMemoryReceiptStorage = new InMemoryReceiptStorage();
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, NullSyncReport.Instance, inMemoryReceiptStorage, RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
Task <BlockHeader[]> buildHeadersResponse = null;
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => buildHeadersResponse = _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
Response blockResponseOptions = Response.AllCorrect;
if (withReceipts)
{
blockResponseOptions |= Response.WithTransactions;
}
Task <BlockBody[]> buildBlocksResponse = null;
syncPeer.GetBlocks(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => buildBlocksResponse = _responseBuilder.BuildBlocksResponse(ci.ArgAt <IList <Keccak> >(0), blockResponseOptions));
syncPeer.GetReceipts(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildReceiptsResponse(buildHeadersResponse.Result, buildBlocksResponse.Result));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = headNumber;
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
Assert.AreEqual(Math.Max(0, headNumber - SyncModeSelector.FullSyncThreshold), _blockTree.BestSuggestedHeader.Number, "headers");
peerInfo.HeadNumber *= 2;
await blockDownloader.DownloadBlocks(peerInfo, 0, CancellationToken.None, downloadOptions);
_blockTree.BestSuggestedHeader.Number.Should().Be(Math.Max(0, headNumber * 2));
_blockTree.IsMainChain(_blockTree.BestSuggestedHeader.Hash).Should().Be(downloadOptions != BlockDownloader.DownloadOptions.DownloadAndProcess);
inMemoryReceiptStorage.Count.Should().Be(withReceipts ? buildBlocksResponse.Result.Sum(b => b.Transactions?.Length ?? 0) : 0);
}
public async Task Peer_sends_just_one_item_when_advertising_more_blocks(long headNumber)
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
syncPeer.GetBlocks(Arg.Any <Keccak[]>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <Keccak[]>(0), Response.AllCorrect | Response.JustFirstHeader));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = headNumber;
Task task = blockDownloader.DownloadBlocks(peerInfo, CancellationToken.None);
await task.ContinueWith(t => Assert.True(t.IsFaulted));
Assert.AreEqual(0, _blockTree.BestSuggested.Number);
}
public async Task Throws_on_receipt_task_exception_when_downloading_receipts(int options, bool shouldThrow)
{
BlockDownloader.DownloadOptions downloadOptions = (BlockDownloader.DownloadOptions)options;
InMemoryReceiptStorage inMemoryReceiptStorage = new InMemoryReceiptStorage();
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, NullSyncReport.Instance, inMemoryReceiptStorage, RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
Task <BlockHeader[]> buildHeadersResponse = null;
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => buildHeadersResponse = _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
Task <BlockBody[]> buildBlocksResponse = null;
syncPeer.GetBlocks(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => buildBlocksResponse = _responseBuilder.BuildBlocksResponse(ci.ArgAt <IList <Keccak> >(0), Response.AllCorrect | Response.WithTransactions));
syncPeer.GetReceipts(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(Task.FromException <TxReceipt[][]>(new TimeoutException()));
PeerInfo peerInfo = new PeerInfo(syncPeer)
{
TotalDifficulty = UInt256.MaxValue, HeadNumber = 1
};
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
peerInfo.HeadNumber *= 2;
Func <Task> action = async() => await blockDownloader.DownloadBlocks(peerInfo, 0, CancellationToken.None, downloadOptions);
if (shouldThrow)
{
action.Should().Throw <EthSynchronizationException>().WithInnerException <AggregateException>().WithInnerException <TimeoutException>();
}
else
{
action.Should().NotThrow();
}
}
public async Task Peer_sends_just_one_item_when_advertising_more_blocks_but_no_bodies(long headNumber)
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, NullSyncReport.Instance, new InMemoryReceiptStorage(), RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect | Response.NoBody));
syncPeer.GetBlocks(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <IList <Keccak> >(0), Response.AllCorrect | Response.JustFirst));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = headNumber;
Task task = blockDownloader.DownloadBlocks(peerInfo, 0, CancellationToken.None);
await task.ContinueWith(t => Assert.False(t.IsFaulted));
Assert.AreEqual(headNumber, _blockTree.BestSuggestedHeader.Number);
}
public async Task Headers_already_known()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect | Response.AllKnown));
syncPeer.GetBlocks(Arg.Any <Keccak[]>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <Keccak[]>(0), Response.AllCorrect | Response.AllKnown));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = 64;
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None)
.ContinueWith(t => Assert.True(t.IsCompletedSuccessfully));
peerInfo.HeadNumber = 128;
await blockDownloader.DownloadBlocks(peerInfo, CancellationToken.None)
.ContinueWith(t => Assert.True(t.IsCompletedSuccessfully));
}
private async Task ExecuteRequest(CancellationToken token, FastBlocksBatch batch)
{
SyncPeerAllocation nodeSyncAllocation = _syncPeerPool.Borrow(BorrowOptions.DoNotReplace | (batch.Prioritized ? BorrowOptions.None : BorrowOptions.LowPriority), "fast blocks", batch.MinNumber);
foreach (PeerInfo peerInfo in _syncPeerPool.UsefulPeers)
{
if (peerInfo.HeadNumber < Math.Max(0, (batch.MinNumber ?? 0) - 1024))
{
if (_logger.IsDebug)
{
_logger.Debug($"Made {peerInfo} sleep for a while - no min number satisfied");
}
_syncPeerPool.ReportNoSyncProgress(peerInfo);
}
}
try
{
ISyncPeer peer = nodeSyncAllocation?.Current?.SyncPeer;
batch.Allocation = nodeSyncAllocation;
if (peer != null)
{
batch.MarkSent();
switch (batch.BatchType)
{
case FastBlocksBatchType.Headers:
{
Task <BlockHeader[]> getHeadersTask = peer.GetBlockHeaders(batch.Headers.StartNumber, batch.Headers.RequestSize, 0, token);
await getHeadersTask.ContinueWith(
t =>
{
if (t.IsCompletedSuccessfully)
{
if (batch.RequestTime > 1000)
{
if (_logger.IsDebug)
{
_logger.Debug($"{batch} - peer is slow {batch.RequestTime:F2}");
}
}
batch.Headers.Response = getHeadersTask.Result;
ValidateHeaders(token, batch);
}
else
{
_syncPeerPool.ReportInvalid(batch.Allocation);
}
}
);
break;
}
case FastBlocksBatchType.Bodies:
{
Task <BlockBody[]> getBodiesTask = peer.GetBlocks(batch.Bodies.Request, token);
await getBodiesTask.ContinueWith(
t =>
{
if (t.IsCompletedSuccessfully)
{
if (batch.RequestTime > 1000)
{
if (_logger.IsDebug)
{
_logger.Debug($"{batch} - peer is slow {batch.RequestTime:F2}");
}
}
batch.Bodies.Response = getBodiesTask.Result;
}
else
{
_syncPeerPool.ReportInvalid(batch.Allocation);
}
}
);
break;
}
case FastBlocksBatchType.Receipts:
{
Task <TxReceipt[][]> getReceiptsTask = peer.GetReceipts(batch.Receipts.Request, token);
await getReceiptsTask.ContinueWith(
t =>
{
if (t.IsCompletedSuccessfully)
{
if (batch.RequestTime > 1000)
{
if (_logger.IsDebug)
{
_logger.Debug($"{batch} - peer is slow {batch.RequestTime:F2}");
}
}
batch.Receipts.Response = getReceiptsTask.Result;
}
else
{
_syncPeerPool.ReportInvalid(batch.Allocation);
}
}
);
break;
}
default:
{
throw new InvalidOperationException($"{nameof(FastBlocksBatchType)} is {batch.BatchType}");
}
}
}
(BlocksDataHandlerResult Result, int ItemsSynced)result = (BlocksDataHandlerResult.InvalidFormat, 0);
try
{
if (batch.Bodies?.Response == null &&
batch.Headers?.Response == null &&
batch.Receipts?.Response == null)
{
// to avoid uncontrolled loop in case of a code error
await Task.Delay(10);
}
result = _fastBlocksFeed.HandleResponse(batch);
}
catch (Exception e)
{
// possibly clear the response and handle empty response batch here (to avoid missing parts)
if (_logger.IsError)
{
_logger.Error($"Error when handling response", e);
}
}
Interlocked.Add(ref _downloadedHeaders, result.ItemsSynced);
if (result.ItemsSynced == 0 && peer != null)
{
_syncPeerPool.ReportNoSyncProgress(nodeSyncAllocation);
}
}
finally
{
if (nodeSyncAllocation != null)
{
_syncPeerPool.Free(nodeSyncAllocation);
}
}
}