public Downloader(DownloaderOptions options)
{
this.options = options;
api = new PluralSightApi(options.Timeout);
queue = new TaskQueue();
queue.ProcessCompleteEvent += ClipDownloaded;
}
public FullSyncFeed(ISyncModeSelector syncModeSelector, ILogManager logManager)
: base(logManager)
{
_syncModeSelector = syncModeSelector ?? throw new ArgumentNullException(nameof(syncModeSelector));
DownloaderOptions options = BuildOptions();
_blocksRequest = new BlocksRequest(options);
_syncModeSelector.Changed += SyncModeSelectorOnChanged;
}
private static async Task RunDownloadOptions(DownloaderOptions options)
{
try
{
var downloader = new Downloader(options);
await downloader.RunAsync();
}
catch (Exception exception)
{
Utils.WriteRedText($"Error occured: {exception.Message}");
}
}
public Downloader(DownloaderOptions options)
{
if (options.Threads > 10)
{
throw new InvalidOperationException();
}
this.options = options;
api = new PluralSightApi(options.Timeout);
queue = new TaskQueue(options.Threads);
queue.ProcessCompleteEvent += ClipDownloaded;
}
public async Task Can_reach_terminal_block(long headNumber, int options, int threshold, bool withBeaconPivot)
{
UInt256 ttd = 10000000;
BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder blockTrees = BlockTreeTests.BlockTreeTestScenario
.GoesLikeThis()
.WithBlockTrees(4, (int)headNumber + 1, true, ttd)
.InsertBeaconPivot(16)
.InsertHeaders(4, 15)
.InsertBeaconBlocks(17, headNumber, BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder.TotalDifficultyMode.Null);
BlockTree notSyncedTree = blockTrees.NotSyncedTree;
BlockTree syncedTree = blockTrees.SyncedTree;
Context ctx = new(notSyncedTree);
DownloaderOptions downloaderOptions = (DownloaderOptions)options;
InMemoryReceiptStorage receiptStorage = new();
MemDb metadataDb = blockTrees.NotSyncedTreeBuilder.MetadataDb;
PoSSwitcher posSwitcher = new(new MergeConfig()
{
Enabled = true, TerminalTotalDifficulty = $"{ttd}"
}, new SyncConfig(), metadataDb, notSyncedTree,
RopstenSpecProvider.Instance, LimboLogs.Instance);
BeaconPivot beaconPivot = new(new SyncConfig(), metadataDb, notSyncedTree, LimboLogs.Instance);
if (withBeaconPivot)
{
beaconPivot.EnsurePivot(blockTrees.SyncedTree.FindHeader(16, BlockTreeLookupOptions.None));
}
MergeBlockDownloader downloader = new(
posSwitcher,
beaconPivot,
ctx.Feed,
ctx.PeerPool,
notSyncedTree,
Always.Valid,
Always.Valid,
NullSyncReport.Instance,
receiptStorage,
RopstenSpecProvider.Instance,
CreateMergePeerChoiceStrategy(posSwitcher, beaconPivot),
new ChainLevelHelper(notSyncedTree, beaconPivot, new SyncConfig(), LimboLogs.Instance),
Substitute.For <ISyncProgressResolver>(),
LimboLogs.Instance);
SyncPeerMock syncPeer = new(syncedTree, false, Response.AllCorrect, 16000000);
PeerInfo peerInfo = new(syncPeer);
await downloader.DownloadBlocks(peerInfo, new BlocksRequest(downloaderOptions), CancellationToken.None);
Assert.True(posSwitcher.HasEverReachedTerminalBlock());
}
private DownloaderOptions BuildOptions()
{
DownloaderOptions options = DownloaderOptions.MoveToMain;
if (_syncConfig.DownloadReceiptsInFastSync)
{
options |= DownloaderOptions.WithReceipts;
}
if (_syncConfig.DownloadBodiesInFastSync)
{
options |= DownloaderOptions.WithBodies;
}
return(options);
}
public Downloader(DownloaderOptions options)
{
this.options = options;
pluralsightService = new PluralsightService();
downloadClient = new DownloadClient(options.Timeout);
}
public override async Task <long> DownloadBlocks(PeerInfo?bestPeer, BlocksRequest blocksRequest,
CancellationToken cancellation)
{
if (_beaconPivot.BeaconPivotExists() == false && _poSSwitcher.HasEverReachedTerminalBlock() == false)
{
return(await base.DownloadBlocks(bestPeer, blocksRequest, cancellation));
}
if (bestPeer == null)
{
string message = $"Not expecting best peer to be null inside the {nameof(BlockDownloader)}";
if (_logger.IsError)
{
_logger.Error(message);
}
throw new ArgumentNullException(message);
}
DownloaderOptions options = blocksRequest.Options;
bool downloadReceipts = (options & DownloaderOptions.WithReceipts) == DownloaderOptions.WithReceipts;
bool shouldProcess = (options & DownloaderOptions.Process) == DownloaderOptions.Process;
bool shouldMoveToMain = (options & DownloaderOptions.MoveToMain) == DownloaderOptions.MoveToMain;
int blocksSynced = 0;
long currentNumber = _blockTree.BestKnownNumber;
if (_logger.IsTrace)
{
_logger.Trace(
$"MergeBlockDownloader GetCurrentNumber: currentNumber {currentNumber}, beaconPivotExists: {_beaconPivot.BeaconPivotExists()}, BestSuggestedBody: {_blockTree.BestSuggestedBody?.Number}, BestKnownNumber: {_blockTree.BestKnownNumber}, BestPeer: {bestPeer}, BestKnownBeaconNumber {_blockTree.BestKnownBeaconNumber}");
}
bool HasMoreToSync(out BlockHeader[]?headers, out int headersToRequest)
{
if (_logger.IsDebug)
{
_logger.Debug($"Continue full sync with {bestPeer} (our best {_blockTree.BestKnownNumber})");
}
headersToRequest = Math.Min(_syncBatchSize.Current, bestPeer.MaxHeadersPerRequest());
if (_logger.IsTrace)
{
_logger.Trace(
$"Full sync request {currentNumber}+{headersToRequest} to peer {bestPeer} with {bestPeer.HeadNumber} blocks. Got {currentNumber} and asking for {headersToRequest} more.");
}
// Note: blocksRequest.NumberOfLatestBlocksToBeIgnored not accounted for
headers = _chainLevelHelper.GetNextHeaders(headersToRequest, bestPeer.HeadNumber);
if (headers == null || headers.Length <= 1)
{
if (_logger.IsTrace)
{
_logger.Trace("Chain level helper got no headers suggestion");
}
return(false);
}
return(true);
}
while (HasMoreToSync(out BlockHeader[]? headers, out int headersToRequest))
{
if (cancellation.IsCancellationRequested)
{
return(blocksSynced); // check before every heavy operation
}
Block[]? blocks = null;
TxReceipt[]?[]? receipts = null;
if (_logger.IsTrace)
{
_logger.Trace(
$"Downloading blocks from peer. CurrentNumber: {currentNumber}, BeaconPivot: {_beaconPivot.PivotNumber}, BestPeer: {bestPeer}, HeaderToRequest: {headersToRequest}");
}
BlockDownloadContext context = new(_specProvider, bestPeer, headers !, downloadReceipts, _receiptsRecovery);
if (cancellation.IsCancellationRequested)
{
return(blocksSynced); // check before every heavy operation
}
await RequestBodies(bestPeer, cancellation, context);
if (downloadReceipts)
{
if (cancellation.IsCancellationRequested)
{
return(blocksSynced); // check before every heavy operation
}
await RequestReceipts(bestPeer, cancellation, context);
}
_sinceLastTimeout++;
if (_sinceLastTimeout > 2)
{
_syncBatchSize.Expand();
}
blocks = context.Blocks;
receipts = context.ReceiptsForBlocks;
if (!(blocks?.Length > 0))
{
if (_logger.IsTrace)
{
_logger.Trace("Break early due to no blocks.");
}
break;
}
for (int blockIndex = 0; blockIndex < blocks.Length; blockIndex++)
{
if (cancellation.IsCancellationRequested)
{
if (_logger.IsTrace)
{
_logger.Trace("Peer sync cancelled");
}
break;
}
Block currentBlock = blocks[blockIndex];
if (_logger.IsTrace)
{
_logger.Trace($"Received {currentBlock} from {bestPeer}");
}
if (currentBlock.IsBodyMissing)
{
throw new EthSyncException($"{bestPeer} didn't send body for block {currentBlock.ToString(Block.Format.Short)}.");
}
// can move this to block tree now?
if (!_blockValidator.ValidateSuggestedBlock(currentBlock))
{
throw new EthSyncException($"{bestPeer} sent an invalid block {currentBlock.ToString(Block.Format.Short)}.");
}
if (shouldProcess)
{
// covering edge case during fastSyncTransition when we're trying to SuggestBlock without the state
bool headIsGenesis = _blockTree.Head?.IsGenesis ?? false;
bool toBeProcessedIsNotBlockOne = currentBlock.Number > 1;
bool isFastSyncTransition = headIsGenesis && toBeProcessedIsNotBlockOne;
if (isFastSyncTransition)
{
long bestFullState = _syncProgressResolver.FindBestFullState();
shouldProcess = currentBlock.Number > bestFullState && bestFullState != 0;
if (!shouldProcess)
{
if (_logger.IsInfo)
{
_logger.Info($"Skipping processing during fastSyncTransition, currentBlock: {currentBlock}, bestFullState: {bestFullState}");
}
downloadReceipts = true;
}
}
}
if (downloadReceipts)
{
TxReceipt[]? contextReceiptsForBlock = receipts ![blockIndex];
if (currentBlock.Header.HasBody && contextReceiptsForBlock == null)
{
throw new EthSyncException($"{bestPeer} didn't send receipts for block {currentBlock.ToString(Block.Format.Short)}.");
}
}
public async Task <long> DownloadBlocks(PeerInfo bestPeer, BlocksRequest blocksRequest, CancellationToken cancellation)
{
if (bestPeer == null)
{
string message = $"Not expecting best peer to be null inside the {nameof(BlockDownloader)}";
if (_logger.IsError)
{
_logger.Error(message);
}
throw new ArgumentNullException(message);
}
var receiptsRecovery = new ReceiptsRecovery(new EthereumEcdsa(_specProvider.ChainId, _logManager), _specProvider);
DownloaderOptions options = blocksRequest.Options;
bool downloadReceipts = (options & DownloaderOptions.WithReceipts) == DownloaderOptions.WithReceipts;
bool shouldProcess = (options & DownloaderOptions.Process) == DownloaderOptions.Process;
bool shouldMoveToMain = (options & DownloaderOptions.MoveToMain) == DownloaderOptions.MoveToMain;
int blocksSynced = 0;
int ancestorLookupLevel = 0;
long currentNumber = Math.Max(0, Math.Min(_blockTree.BestKnownNumber, bestPeer.HeadNumber - 1));
// pivot number - 6 for uncle validation
// long currentNumber = Math.Max(Math.Max(0, pivotNumber - 6), Math.Min(_blockTree.BestKnownNumber, bestPeer.HeadNumber - 1));
while (bestPeer.TotalDifficulty > (_blockTree.BestSuggestedHeader?.TotalDifficulty ?? 0) && currentNumber <= bestPeer.HeadNumber)
{
if (_logger.IsDebug)
{
_logger.Debug($"Continue full sync with {bestPeer} (our best {_blockTree.BestKnownNumber})");
}
long blocksLeft = bestPeer.HeadNumber - currentNumber - (blocksRequest.NumberOfLatestBlocksToBeIgnored ?? 0);
int headersToRequest = (int)Math.Min(blocksLeft + 1, _syncBatchSize.Current);
if (headersToRequest <= 1)
{
break;
}
headersToRequest = Math.Min(headersToRequest, bestPeer.MaxHeadersPerRequest());
if (_logger.IsTrace)
{
_logger.Trace($"Full sync request {currentNumber}+{headersToRequest} to peer {bestPeer} with {bestPeer.HeadNumber} blocks. Got {currentNumber} and asking for {headersToRequest} more.");
}
if (cancellation.IsCancellationRequested)
{
return(blocksSynced); // check before every heavy operation
}
BlockHeader[] headers = await RequestHeaders(bestPeer, cancellation, currentNumber, headersToRequest);
BlockDownloadContext context = new(_specProvider, bestPeer, headers, downloadReceipts, receiptsRecovery);
if (cancellation.IsCancellationRequested)
{
return(blocksSynced); // check before every heavy operation
}
await RequestBodies(bestPeer, cancellation, context);
if (downloadReceipts)
{
if (cancellation.IsCancellationRequested)
{
return(blocksSynced); // check before every heavy operation
}
await RequestReceipts(bestPeer, cancellation, context);
}
_sinceLastTimeout++;
if (_sinceLastTimeout > 2)
{
_syncBatchSize.Expand();
}
Block[] blocks = context.Blocks;
Block blockZero = blocks[0];
if (context.FullBlocksCount > 0)
{
bool parentIsKnown = _blockTree.IsKnownBlock(blockZero.Number - 1, blockZero.ParentHash);
if (!parentIsKnown)
{
ancestorLookupLevel++;
if (ancestorLookupLevel >= _ancestorJumps.Length)
{
if (_logger.IsWarn)
{
_logger.Warn($"Could not find common ancestor with {bestPeer}");
}
throw new EthSyncException("Peer with inconsistent chain in sync");
}
int ancestorJump = _ancestorJumps[ancestorLookupLevel] - _ancestorJumps[ancestorLookupLevel - 1];
currentNumber = currentNumber >= ancestorJump ? (currentNumber - ancestorJump) : 0L;
continue;
}
}
ancestorLookupLevel = 0;
for (int blockIndex = 0; blockIndex < context.FullBlocksCount; blockIndex++)
{
if (cancellation.IsCancellationRequested)
{
if (_logger.IsTrace)
{
_logger.Trace("Peer sync cancelled");
}
break;
}
Block currentBlock = blocks[blockIndex];
if (_logger.IsTrace)
{
_logger.Trace($"Received {currentBlock} from {bestPeer}");
}
// can move this to block tree now?
if (!_blockValidator.ValidateSuggestedBlock(currentBlock))
{
throw new EthSyncException($"{bestPeer} sent an invalid block {currentBlock.ToString(Block.Format.Short)}.");
}
if (downloadReceipts)
{
TxReceipt[]? contextReceiptsForBlock = context.ReceiptsForBlocks ![blockIndex];
public BlocksRequest(DownloaderOptions options)
{
Options = options;
}
public BlocksRequest(DownloaderOptions options, int?numberOfLatestBlocksToBeIgnored)
{
Options = options;
NumberOfLatestBlocksToBeIgnored = numberOfLatestBlocksToBeIgnored;
}
public async Task Merge_Happy_path(long pivot, long headNumber, int options, int threshold, long insertedBeaconBlocks)
{
BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder blockTrees = BlockTreeTests.BlockTreeTestScenario
.GoesLikeThis()
.WithBlockTrees(4, (int)headNumber + 1)
.InsertBeaconPivot(pivot)
.InsertHeaders(4, pivot - 1)
.InsertBeaconBlocks(pivot + 1, insertedBeaconBlocks, BlockTreeTests.BlockTreeTestScenario.ScenarioBuilder.TotalDifficultyMode.Null);
BlockTree notSyncedTree = blockTrees.NotSyncedTree;
BlockTree syncedTree = blockTrees.SyncedTree;
Context ctx = new(notSyncedTree);
DownloaderOptions downloaderOptions = (DownloaderOptions)options;
bool withReceipts = downloaderOptions == DownloaderOptions.WithReceipts;
InMemoryReceiptStorage receiptStorage = new();
MemDb metadataDb = blockTrees.NotSyncedTreeBuilder.MetadataDb;
PoSSwitcher posSwitcher = new(new MergeConfig()
{
Enabled = true, TerminalTotalDifficulty = "0"
}, new SyncConfig(), metadataDb, notSyncedTree,
RopstenSpecProvider.Instance, LimboLogs.Instance);
BeaconPivot beaconPivot = new(new SyncConfig(), metadataDb, notSyncedTree, LimboLogs.Instance);
beaconPivot.EnsurePivot(blockTrees.SyncedTree.FindHeader(pivot, BlockTreeLookupOptions.None));
beaconPivot.ProcessDestination = blockTrees.SyncedTree.FindHeader(pivot, BlockTreeLookupOptions.None);
MergeBlockDownloader downloader = new(
posSwitcher,
beaconPivot,
ctx.Feed,
ctx.PeerPool,
notSyncedTree,
Always.Valid,
Always.Valid,
NullSyncReport.Instance,
receiptStorage,
RopstenSpecProvider.Instance,
CreateMergePeerChoiceStrategy(posSwitcher, beaconPivot),
new ChainLevelHelper(notSyncedTree, beaconPivot, new SyncConfig(), LimboLogs.Instance),
Substitute.For <ISyncProgressResolver>(),
LimboLogs.Instance);
Response responseOptions = Response.AllCorrect;
if (withReceipts)
{
responseOptions |= Response.WithTransactions;
}
SyncPeerMock syncPeer = new(syncedTree, withReceipts, responseOptions, 16000000);
PeerInfo peerInfo = new(syncPeer);
await downloader.DownloadBlocks(peerInfo, new BlocksRequest(downloaderOptions), CancellationToken.None);
ctx.BlockTree.BestSuggestedHeader.Number.Should().Be(Math.Max(0, insertedBeaconBlocks));
ctx.BlockTree.BestKnownNumber.Should().Be(Math.Max(0, insertedBeaconBlocks));
int receiptCount = 0;
for (int i = (int)Math.Max(0, headNumber - threshold); i < peerInfo.HeadNumber; i++)
{
if (i % 3 == 0)
{
receiptCount += 2;
}
}
receiptStorage.Count.Should().Be(withReceipts ? receiptCount : 0);
beaconPivot.ProcessDestination?.Number.Should().Be(insertedBeaconBlocks);
}
