private void StartFastSyncComponents()
{
_fastSyncFeed = new FastSyncFeed(_syncMode, _syncConfig, _logManager);
BlockDownloader downloader = new BlockDownloader(_fastSyncFeed, _syncPeerPool, _blockTree, _blockValidator, _sealValidator, _syncReport, _receiptStorage, _specProvider, _logManager);
downloader.SyncEvent += DownloaderOnSyncEvent;
downloader.Start(_syncCancellation.Token).ContinueWith(t =>
{
if (t.IsFaulted)
{
if (_logger.IsError)
{
_logger.Error("Fast sync failed", t.Exception);
}
}
else
{
if (_logger.IsInfo)
{
_logger.Info("Fast sync blocks downloader task completed.");
}
}
});
}
public async Task Can_cancel_adding_headers()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, new SlowHeaderValidator(), TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
syncPeer.GetBlocks(Arg.Any <Keccak[]>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <Keccak[]>(0), Response.AllCorrect));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = 1000;
CancellationTokenSource cancellation = new CancellationTokenSource();
cancellation.CancelAfter(1000);
Task task = blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, cancellation.Token);
await task.ContinueWith(t => Assert.True(t.IsCanceled, "headers"));
peerInfo.HeadNumber *= 2;
cancellation = new CancellationTokenSource();
cancellation.CancelAfter(1000);
task = blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, cancellation.Token);
await task.ContinueWith(t => Assert.True(t.IsCanceled, "blocks"));
}
public async Task Happy_path(long headNumber)
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
syncPeer.GetBlocks(Arg.Any <Keccak[]>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <Keccak[]>(0), Response.AllCorrect));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = headNumber;
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
Assert.AreEqual(Math.Max(0, headNumber - SyncModeSelector.FullSyncThreshold), _blockTree.BestSuggested.Number, "headers");
peerInfo.HeadNumber *= 2;
await blockDownloader.DownloadBlocks(peerInfo, CancellationToken.None);
Assert.AreEqual(Math.Max(0, headNumber * 2), _blockTree.BestSuggested.Number);
}
public async Task Can_sync_with_peer_when_it_times_out_on_full_batch()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(async ci => await _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect | Response.TimeoutOnFullBatch));
syncPeer.GetBlocks(Arg.Any <Keccak[]>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <Keccak[]>(0), Response.AllCorrect | Response.TimeoutOnFullBatch));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = SyncBatchSize.Max * 2 + 32;
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None).ContinueWith(t => { });
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
Assert.AreEqual(Math.Max(0, peerInfo.HeadNumber - SyncModeSelector.FullSyncThreshold), _blockTree.BestSuggested.Number);
peerInfo.HeadNumber *= 2;
await blockDownloader.DownloadBlocks(peerInfo, CancellationToken.None).ContinueWith(t => { });
await blockDownloader.DownloadBlocks(peerInfo, CancellationToken.None);
Assert.AreEqual(Math.Max(0, peerInfo.HeadNumber), _blockTree.BestSuggested.Number);
}
public async Task Can_cancel_seal_validation()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, new SlowSealValidator(), NullSyncReport.Instance, new InMemoryReceiptStorage(), RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
syncPeer.GetBlocks(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <IList <Keccak> >(0), Response.AllCorrect));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = 1000;
CancellationTokenSource cancellation = new CancellationTokenSource();
cancellation.CancelAfter(1000);
Task task = blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, cancellation.Token);
await task.ContinueWith(t => Assert.True(t.IsCanceled, $"headers {t.Status}"));
peerInfo.HeadNumber = 2000;
cancellation = new CancellationTokenSource();
cancellation.CancelAfter(1000);
task = blockDownloader.DownloadBlocks(peerInfo, 0, cancellation.Token);
await task.ContinueWith(t => Assert.True(t.IsCanceled, $"blocks {t.Status}"));
}
public async Task Throws_on_incorrect_receipts_root()
{
InMemoryReceiptStorage inMemoryReceiptStorage = new InMemoryReceiptStorage();
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, NullSyncReport.Instance, inMemoryReceiptStorage, RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
Task <BlockHeader[]> buildHeadersResponse = null;
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => buildHeadersResponse = _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
Task <BlockBody[]> buildBlocksResponse = null;
syncPeer.GetBlocks(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => buildBlocksResponse = _responseBuilder.BuildBlocksResponse(ci.ArgAt <IList <Keccak> >(0), Response.AllCorrect | Response.WithTransactions));
syncPeer.GetReceipts(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildReceiptsResponse(buildHeadersResponse.Result, buildBlocksResponse.Result, Response.IncorrectReceiptRoot).Result);
PeerInfo peerInfo = new PeerInfo(syncPeer)
{
TotalDifficulty = UInt256.MaxValue, HeadNumber = 1
};
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
peerInfo.HeadNumber *= 2;
Func <Task> action = async() => await blockDownloader.DownloadBlocks(peerInfo, 0, CancellationToken.None, BlockDownloader.DownloadOptions.DownloadWithReceipts);
action.Should().Throw <EthSynchronizationException>();
}
public async Task Throws_on_null_best_peer()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
Task task1 = blockDownloader.DownloadHeaders(null, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
await task1.ContinueWith(t => Assert.True(t.IsFaulted));
Task task2 = blockDownloader.DownloadBlocks(null, CancellationToken.None);
await task2.ContinueWith(t => Assert.True(t.IsFaulted));
}
public async Task Faults_on_get_headers_faulting()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = new ThrowingPeer();
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = 1000;
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None)
.ContinueWith(t => Assert.True(t.IsFaulted));
}
public async Task Throws_on_inconsistent_batch()
{
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect ^ Response.Consistent));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = 1024;
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
Task task = blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
await task.ContinueWith(t => Assert.True(t.IsFaulted));
}
public async Task Throws_on_invalid_header()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.NeverValid, TestSealValidator.AlwaysValid, NullSyncReport.Instance, new InMemoryReceiptStorage(), RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = 1000;
Task task = blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
await task.ContinueWith(t => Assert.True(t.IsFaulted));
}
public async Task Happy_path(long headNumber, int options)
{
BlockDownloader.DownloadOptions downloadOptions = (BlockDownloader.DownloadOptions)options;
bool withReceipts = downloadOptions == BlockDownloader.DownloadOptions.DownloadWithReceipts;
InMemoryReceiptStorage inMemoryReceiptStorage = new InMemoryReceiptStorage();
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, NullSyncReport.Instance, inMemoryReceiptStorage, RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
Task <BlockHeader[]> buildHeadersResponse = null;
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => buildHeadersResponse = _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
Response blockResponseOptions = Response.AllCorrect;
if (withReceipts)
{
blockResponseOptions |= Response.WithTransactions;
}
Task <BlockBody[]> buildBlocksResponse = null;
syncPeer.GetBlocks(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => buildBlocksResponse = _responseBuilder.BuildBlocksResponse(ci.ArgAt <IList <Keccak> >(0), blockResponseOptions));
syncPeer.GetReceipts(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildReceiptsResponse(buildHeadersResponse.Result, buildBlocksResponse.Result));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = headNumber;
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
Assert.AreEqual(Math.Max(0, headNumber - SyncModeSelector.FullSyncThreshold), _blockTree.BestSuggestedHeader.Number, "headers");
peerInfo.HeadNumber *= 2;
await blockDownloader.DownloadBlocks(peerInfo, 0, CancellationToken.None, downloadOptions);
_blockTree.BestSuggestedHeader.Number.Should().Be(Math.Max(0, headNumber * 2));
_blockTree.IsMainChain(_blockTree.BestSuggestedHeader.Hash).Should().Be(downloadOptions != BlockDownloader.DownloadOptions.DownloadAndProcess);
inMemoryReceiptStorage.Count.Should().Be(withReceipts ? buildBlocksResponse.Result.Sum(b => b.Transactions?.Length ?? 0) : 0);
}
public async Task Peer_sends_just_one_item_when_advertising_more_blocks(long headNumber)
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
syncPeer.GetBlocks(Arg.Any <Keccak[]>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <Keccak[]>(0), Response.AllCorrect | Response.JustFirstHeader));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = headNumber;
Task task = blockDownloader.DownloadBlocks(peerInfo, CancellationToken.None);
await task.ContinueWith(t => Assert.True(t.IsFaulted));
Assert.AreEqual(0, _blockTree.BestSuggested.Number);
}
public async Task Throws_on_receipt_task_exception_when_downloading_receipts(int options, bool shouldThrow)
{
BlockDownloader.DownloadOptions downloadOptions = (BlockDownloader.DownloadOptions)options;
InMemoryReceiptStorage inMemoryReceiptStorage = new InMemoryReceiptStorage();
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, NullSyncReport.Instance, inMemoryReceiptStorage, RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
Task <BlockHeader[]> buildHeadersResponse = null;
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => buildHeadersResponse = _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect));
Task <BlockBody[]> buildBlocksResponse = null;
syncPeer.GetBlocks(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => buildBlocksResponse = _responseBuilder.BuildBlocksResponse(ci.ArgAt <IList <Keccak> >(0), Response.AllCorrect | Response.WithTransactions));
syncPeer.GetReceipts(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(Task.FromException <TxReceipt[][]>(new TimeoutException()));
PeerInfo peerInfo = new PeerInfo(syncPeer)
{
TotalDifficulty = UInt256.MaxValue, HeadNumber = 1
};
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None);
peerInfo.HeadNumber *= 2;
Func <Task> action = async() => await blockDownloader.DownloadBlocks(peerInfo, 0, CancellationToken.None, downloadOptions);
if (shouldThrow)
{
action.Should().Throw <EthSynchronizationException>().WithInnerException <AggregateException>().WithInnerException <TimeoutException>();
}
else
{
action.Should().NotThrow();
}
}
public async Task Peer_sends_just_one_item_when_advertising_more_blocks_but_no_bodies(long headNumber)
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, NullSyncReport.Instance, new InMemoryReceiptStorage(), RopstenSpecProvider.Instance, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect | Response.NoBody));
syncPeer.GetBlocks(Arg.Any <IList <Keccak> >(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <IList <Keccak> >(0), Response.AllCorrect | Response.JustFirst));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = headNumber;
Task task = blockDownloader.DownloadBlocks(peerInfo, 0, CancellationToken.None);
await task.ContinueWith(t => Assert.False(t.IsFaulted));
Assert.AreEqual(headNumber, _blockTree.BestSuggestedHeader.Number);
}
public async Task Headers_already_known()
{
BlockDownloader blockDownloader = new BlockDownloader(_blockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, LimboLogs.Instance);
ISyncPeer syncPeer = Substitute.For <ISyncPeer>();
syncPeer.GetBlockHeaders(Arg.Any <long>(), Arg.Any <int>(), Arg.Any <int>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildHeaderResponse(ci.ArgAt <long>(0), ci.ArgAt <int>(1), Response.AllCorrect | Response.AllKnown));
syncPeer.GetBlocks(Arg.Any <Keccak[]>(), Arg.Any <CancellationToken>())
.Returns(ci => _responseBuilder.BuildBlocksResponse(ci.ArgAt <Keccak[]>(0), Response.AllCorrect | Response.AllKnown));
PeerInfo peerInfo = new PeerInfo(syncPeer);
peerInfo.TotalDifficulty = UInt256.MaxValue;
peerInfo.HeadNumber = 64;
await blockDownloader.DownloadHeaders(peerInfo, SyncModeSelector.FullSyncThreshold, CancellationToken.None)
.ContinueWith(t => Assert.True(t.IsCompletedSuccessfully));
peerInfo.HeadNumber = 128;
await blockDownloader.DownloadBlocks(peerInfo, CancellationToken.None)
.ContinueWith(t => Assert.True(t.IsCompletedSuccessfully));
}
/// <summary>
/// Starts the file download.
/// </summary>
public void StartDownload()
{
Trace.TraceInformation($"[{URL}] Downloading.");
if (!FileExists)
{
throw new FileNotFoundException("The file to download does not exist.", URL.ToString());
}
int intConnectionsToUse = _metadata.SupportsResume ? _maxConnections : 1;
if (ServicePointManager.DefaultConnectionLimit < 1)
{
throw new Exception(string.Format("Only {0} connections can be created to the same file; {1} are wanted.", ServicePointManager.DefaultConnectionLimit, 1));
}
if (ServicePointManager.DefaultConnectionLimit < intConnectionsToUse)
{
intConnectionsToUse = ServicePointManager.DefaultConnectionLimit;
}
//get the list of ranges we have not already downloaded
RangeSet rgsMissingRanges = new RangeSet();
rgsMissingRanges.AddRange(new Range(0, _metadata.Length - 1));
if (File.Exists(_fileMetadataPath))
{
string[] strRanges = File.ReadAllLines(_fileMetadataPath);
foreach (string strRange in strRanges)
{
string strCleanRange = strRange.Trim().Trim('\0');
if (string.IsNullOrEmpty(strCleanRange))
{
continue;
}
rgsMissingRanges.RemoveRange(Range.Parse(strCleanRange));
}
}
else if (File.Exists(_savePath))
{
File.Delete(_savePath);
}
int intMinBlockSize = (int)Math.Min((ulong)_minBlockSize, rgsMissingRanges.TotalSize);
int intBaseBlockSize = (int)Math.Max(rgsMissingRanges.TotalSize / (ulong)intConnectionsToUse, (ulong)intMinBlockSize);
if (intConnectionsToUse > 1)
{
intBaseBlockSize = Math.Min(intBaseBlockSize, MaxBlockSize);
}
//break the ranges into blocks to be downloaded
foreach (Range rngNeeded in rgsMissingRanges)
{
//find out how many blocks will fit into the range
int intBlockCount = (int)(rngNeeded.Size / (ulong)intBaseBlockSize);
if (intBlockCount == 0)
{
intBlockCount = 1;
}
//there is likely to be some remainder (there are likely a fractional number of blocks
// in the range), so lets distrubute the remainder amongst all of the blocks
// we do this by elarging our blocksize
ulong intBlockSize = (ulong)Math.Ceiling(rngNeeded.Size / (double)intBlockCount);
ulong intBlockStart = rngNeeded.StartByte;
for (; intBlockStart + intBlockSize < rngNeeded.EndByte; intBlockStart += intBlockSize)
{
_requiredBlocks.Enqueue(new Range(intBlockStart, intBlockStart + intBlockSize - 1));
}
_requiredBlocks.Enqueue(new Range(intBlockStart, rngNeeded.EndByte));
}
_writer = new FileWriter(_savePath, _fileMetadataPath);
_startTime = DateTime.Now;
//spawn the downloading threads
int intRequiredBlocks = _requiredBlocks.Count;
lock (_downloaders)
{
for (int i = 0; i < (intRequiredBlocks < intConnectionsToUse ? intRequiredBlocks : intConnectionsToUse); i++)
{
BlockDownloader bdrDownloader = new BlockDownloader(this, _metadata, _writer, _writeBufferSize, _userAgent);
bdrDownloader.FinishedDownloading += new EventHandler(Downloader_FinishedDownloading);
bdrDownloader.Start();
_downloaders.Add(bdrDownloader);
}
}
}
/// <summary>
/// Starts the file download.
/// </summary>
public void StartDownload()
{
Trace.TraceInformation(String.Format("[{0}] Downloading.", m_uriURL.ToString()));
if (!FileExists)
throw new FileNotFoundException("The file to download does not exist.", m_uriURL.ToString());
Int32 intConnectionsToUse = m_fmdInfo.SupportsResume ? m_intMaxConnections : 1;
if (ServicePointManager.DefaultConnectionLimit < 1)
throw new Exception(String.Format("Only {0} connections can be created to the same file; {1} are wanted.", ServicePointManager.DefaultConnectionLimit, 1));
else if (ServicePointManager.DefaultConnectionLimit < intConnectionsToUse)
intConnectionsToUse = ServicePointManager.DefaultConnectionLimit;
//get the list of ranges we have not already downloaded
RangeSet rgsMissingRanges = new RangeSet();
rgsMissingRanges.AddRange(new Range(0, m_fmdInfo.Length - 1));
if (File.Exists(m_strFileMetadataPath))
{
string[] strRanges = File.ReadAllLines(m_strFileMetadataPath);
foreach (string strRange in strRanges)
{
string strCleanRange = strRange.Trim().Trim('\0');
if (String.IsNullOrEmpty(strCleanRange))
continue;
rgsMissingRanges.RemoveRange(Range.Parse(strCleanRange));
}
}
else if (File.Exists(m_strSavePath))
File.Delete(m_strSavePath);
Int32 intMinBlockSize = (Int32)Math.Min((UInt64)m_intMinBlockSize, rgsMissingRanges.TotalSize);
Int32 intBaseBlockSize = (Int32)Math.Max(rgsMissingRanges.TotalSize / (UInt64)intConnectionsToUse, (UInt64)intMinBlockSize);
if (intConnectionsToUse > 1)
intBaseBlockSize = Math.Min(intBaseBlockSize, m_intMaxBlockSize);
//break the ranges into blocks to be downloaded
foreach (Range rngNeeded in rgsMissingRanges)
{
//find out how many blocks will fit into the range
Int32 intBlockCount = (Int32)(rngNeeded.Size / (UInt64)intBaseBlockSize);
if (intBlockCount == 0)
intBlockCount = 1;
//there is likely to be some remainder (there are likely a fractional number of blocks
// in the range), so lets distrubute the remainder amongst all of the blocks
// we do this by elarging our blocksize
UInt64 intBlockSize = (UInt64)Math.Ceiling(rngNeeded.Size / (double)intBlockCount);
UInt64 intBlockStart = rngNeeded.StartByte;
for (; intBlockStart + intBlockSize < rngNeeded.EndByte; intBlockStart += intBlockSize)
m_queRequiredBlocks.Enqueue(new Range(intBlockStart, intBlockStart + intBlockSize - 1));
m_queRequiredBlocks.Enqueue(new Range(intBlockStart, rngNeeded.EndByte));
}
m_fwrWriter = new FileWriter(m_strSavePath, m_strFileMetadataPath);
m_dteStartTime = DateTime.Now;
//spawn the downloading threads
Int32 intRequiredBlocks = m_queRequiredBlocks.Count;
lock (m_lstDownloaders)
{
for (Int32 i = 0; i < (intRequiredBlocks < intConnectionsToUse ? intRequiredBlocks : intConnectionsToUse); i++)
{
BlockDownloader bdrDownloader = new BlockDownloader(this, m_fmdInfo, m_fwrWriter, m_intWriteBufferSize, m_strUserAgent);
bdrDownloader.FinishedDownloading += new EventHandler(Downloader_FinishedDownloading);
bdrDownloader.Start();
m_lstDownloaders.Add(bdrDownloader);
}
}
}
public async Task TestServicesAsync(string networkString)
{
var network = Network.GetNetwork(networkString);
var blocksToDownload = new HashSet <uint256>();
if (network == Network.Main)
{
blocksToDownload.Add(new uint256("00000000000000000037c2de35bd85f3e57f14ddd741ce6cee5b28e51473d5d0"));
blocksToDownload.Add(new uint256("000000000000000000115315a43cb0cdfc4ea54a0e92bed127f4e395e718d8f9"));
blocksToDownload.Add(new uint256("00000000000000000011b5b042ad0522b69aae36f7de796f563c895714bbd629"));
}
else if (network == Network.TestNet)
{
blocksToDownload.Add(new uint256("0000000097a664c4084b49faa6fd4417055cb8e5aac480abc31ddc57a8208524"));
blocksToDownload.Add(new uint256("000000009ed5b82259ecd2aa4cd1f119db8da7a70e7ea78d9c9f603e01f93bcc"));
blocksToDownload.Add(new uint256("00000000e6da8c2da304e9f5ad99c079df2c3803b49efded3061ecaf206ddc66"));
}
else
{
throw new NotSupportedException(network.ToString());
}
var manager = KeyManager.CreateNew(out Mnemonic mnemonic, "password");
var dataFolder = Path.Combine(SharedFixture.DataDir, nameof(TestServicesAsync));
Directory.CreateDirectory(SharedFixture.DataDir);
var addressManagerFilePath = Path.Combine(SharedFixture.DataDir, $"AddressManager{network}.dat");
var blocksFolderPath = Path.Combine(SharedFixture.DataDir, $"Blocks{network}");
var connectionParameters = new NodeConnectionParameters();
AddressManager addressManager = null;
BlockDownloader downloader = null;
try
{
try
{
addressManager = AddressManager.LoadPeerFile(addressManagerFilePath);
Logger.LogInfo <WalletService>($"Loaded {nameof(AddressManager)} from `{addressManagerFilePath}`.");
}
catch (FileNotFoundException ex)
{
Logger.LogInfo <WalletService>($"{nameof(AddressManager)} did not exist at `{addressManagerFilePath}`. Initializing new one.");
Logger.LogTrace <WalletService>(ex);
addressManager = new AddressManager();
}
connectionParameters.TemplateBehaviors.Add(new AddressManagerBehavior(addressManager));
var memPoolService = new MemPoolService();
connectionParameters.TemplateBehaviors.Add(new MemPoolBehavior(memPoolService));
using (var nodes = new NodesGroup(network, connectionParameters,
new NodeRequirement
{
RequiredServices = NodeServices.Network,
MinVersion = ProtocolVersion.WITNESS_VERSION
}))
{
downloader = new BlockDownloader(nodes, blocksFolderPath);
Assert.True(Directory.Exists(blocksFolderPath));
downloader.Start();
foreach (var hash in blocksToDownload)
{
downloader.QueToDownload(hash);
}
var wallet = new WalletService(dataFolder, network, manager, nodes, memPoolService);
try
{
nodes.ConnectedNodes.Added += ConnectedNodes_Added;
nodes.ConnectedNodes.Removed += ConnectedNodes_Removed;
memPoolService.TransactionReceived += MemPoolService_TransactionReceived;
nodes.Connect();
// Using the interlocked, not because it makes sense in this context, but to
// set an example that these values are often concurrency sensitive
var times = 0;
while (Interlocked.Read(ref _nodeCount) < 3)
{
if (times > 4200) // 7 minutes
{
throw new TimeoutException($"Connection test timed out.");
}
await Task.Delay(100);
times++;
}
times = 0;
while (Interlocked.Read(ref _mempoolTransactionCount) < 3)
{
if (times > 3000) // 3 minutes
{
throw new TimeoutException($"{nameof(MemPoolService)} test timed out.");
}
await Task.Delay(100);
times++;
}
foreach (var hash in blocksToDownload)
{
times = 0;
while (downloader.GetBlock(hash) == null)
{
if (times > 1800) // 3 minutes
{
throw new TimeoutException($"{nameof(BlockDownloader)} test timed out.");
}
await Task.Delay(100);
times++;
}
Assert.True(File.Exists(Path.Combine(blocksFolderPath, hash.ToString())));
Logger.LogInfo <WalletTests>($"Full block is downloaded: {hash}.");
}
}
finally
{
nodes.ConnectedNodes.Added -= ConnectedNodes_Added;
nodes.ConnectedNodes.Removed -= ConnectedNodes_Removed;
memPoolService.TransactionReceived -= MemPoolService_TransactionReceived;
}
}
}
finally
{
downloader?.Stop();
// So next test will download the block.
foreach (var hash in blocksToDownload)
{
downloader.TryRemove(hash);
}
Directory.Delete(blocksFolderPath, recursive: true);
addressManager?.SavePeerFile(addressManagerFilePath, network);
Logger.LogInfo <WalletTests>($"Saved {nameof(AddressManager)} to `{addressManagerFilePath}`.");
}
}