/// <inheritdoc/>
public override async Task <InnerStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
{
var batchSize = BlockStoreInnerStepContext.DownloadStackThreshold - context.DownloadStack.Count;
var batchList = new List <ChainedHeader>(batchSize);
while (batchList.Count < batchSize)
{
if (await this.ShouldStopFindingBlocksAsync(context))
{
context.StopFindingBlocks();
break;
}
batchList.Add(context.NextChainedHeader);
context.DownloadStack.Enqueue(context.NextChainedHeader);
context.GetNextBlock();
}
if (batchList.Any())
{
this.logger.LogTrace("{0} blocks requested to be downloaded by the puller.", batchList.Count);
context.BlockStoreLoop.BlockPuller.AskForMultipleBlocks(batchList.ToArray());
}
return(InnerStepResult.Next);
}
/// <inheritdoc/>
internal override async Task <BlockStoreLoopStepResult> ExecuteAsync(ChainedBlock nextChainedBlock, CancellationToken token, bool disposeMode)
{
if (disposeMode)
{
return(BlockStoreLoopStepResult.Break());
}
var result = BlockStoreLoopStepResult.Next();
var context = new BlockStoreInnerStepContext(token, this.BlockStoreLoop).Initialize(nextChainedBlock);
this.BlockStoreLoop.BlockPuller.AskBlock(nextChainedBlock);
while (!token.IsCancellationRequested)
{
foreach (var item in context.Routine.ToList())
{
var executionResult = await item.ExecuteAsync(context);
if (executionResult.ShouldBreak)
{
return(result);
}
}
}
return(result);
}
private async Task <bool> ShouldStopFindingBlocks(BlockStoreInnerStepContext context)
{
if (context.NextChainedBlock == null)
{
return(true);
}
if (context.DownloadStack.Count >= BlockStoreInnerStepContext.DownloadStackThreshold)
{
return(true);
}
if (context.InputChainedBlock != null && (context.NextChainedBlock.Header.HashPrevBlock != context.InputChainedBlock.HashBlock))
{
return(true);
}
if (context.NextChainedBlock.Height > context.BlockStoreLoop.ChainState.HighestValidatedPoW?.Height)
{
return(true);
}
if (context.BlockStoreLoop.PendingStorage.ContainsKey(context.NextChainedBlock.HashBlock))
{
return(true);
}
if (await context.BlockStoreLoop.BlockRepository.ExistAsync(context.NextChainedBlock.HashBlock))
{
return(true);
}
return(false);
}
/// <inheritdoc/>
public override async Task <BlockStoreLoopStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
{
context.GetNextBlock();
if (await ShouldStopFindingBlocks(context))
{
if (!context.DownloadStack.Any())
{
return(BlockStoreLoopStepResult.Break());
}
context.StopFindingBlocks();
}
else
{
context.BlockStoreLoop.BlockPuller.AskBlock(context.NextChainedBlock);
context.DownloadStack.Enqueue(context.NextChainedBlock);
if (context.DownloadStack.Count == context.BlockStoreLoop.BatchDownloadSize)
{
context.StopFindingBlocks();
}
}
return(BlockStoreLoopStepResult.Next());
}
/// <inheritdoc/>
internal override async Task <StepResult> ExecuteAsync(ChainedBlock nextChainedBlock, CancellationToken token, bool disposeMode)
{
this.logger.LogTrace("({0}:'{1}/{2}',{3}:{4})", nameof(nextChainedBlock), nextChainedBlock?.HashBlock, nextChainedBlock?.Height, nameof(disposeMode), disposeMode);
if (disposeMode)
{
this.logger.LogTrace("(-):{0}", StepResult.Stop);
return(StepResult.Stop);
}
var context = new BlockStoreInnerStepContext(token, this.BlockStoreLoop, nextChainedBlock, this.loggerFactory, this.dateTimeProvider);
while (!token.IsCancellationRequested)
{
foreach (var innerStep in context.InnerSteps.ToList())
{
InnerStepResult innerStepResult = await innerStep.ExecuteAsync(context);
if (innerStepResult == InnerStepResult.Stop)
{
this.logger.LogTrace("(-):{0}", StepResult.Next);
return(StepResult.Next);
}
}
}
this.logger.LogTrace("(-):{0}", StepResult.Next);
return(StepResult.Next);
}
private async Task <bool> ShouldStopFindingBlocks(BlockStoreInnerStepContext context)
{
if (context.NextChainedBlock == null)
{
this.logger.LogTrace("{0} is null", nameof(context.NextChainedBlock));
return(true);
}
if (context.InputChainedBlock != null && (context.NextChainedBlock.Header.HashPrevBlock != context.InputChainedBlock.HashBlock))
{
this.logger.LogTrace("{0} != {1}", nameof(context.NextChainedBlock.Header.HashPrevBlock), nameof(context.InputChainedBlock.HashBlock));
return(true);
}
if (context.NextChainedBlock.Height > context.BlockStoreLoop.ChainState.HighestValidatedPoW?.Height)
{
this.logger.LogTrace("{0} height > {1} height", nameof(context.NextChainedBlock), nameof(context.BlockStoreLoop.ChainState.HighestValidatedPoW));
return(true);
}
if (context.BlockStoreLoop.PendingStorage.ContainsKey(context.NextChainedBlock.HashBlock))
{
this.logger.LogTrace("{0}='{1}/{2}' exists in pending storage.", nameof(context.NextChainedBlock), context.NextChainedBlock.HashBlock, context.NextChainedBlock.Height);
return(true);
}
if (await context.BlockStoreLoop.BlockRepository.ExistAsync(context.NextChainedBlock.HashBlock))
{
this.logger.LogTrace("{0}='{1}/{2}' exists in the repository.", nameof(context.NextChainedBlock), context.NextChainedBlock.HashBlock, context.NextChainedBlock.Height);
return(true);
}
return(false);
}
/// <inheritdoc/>
public override async Task <InnerStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
{
if (!context.DownloadStack.Any())
{
this.logger.LogTrace("(-)[EMPTY_STACK1]:{0}", InnerStepResult.Stop);
return(InnerStepResult.Stop);
}
while (context.BlocksPushedCount <= BlockStoreInnerStepContext.DownloadStackPushThreshold)
{
DownloadedBlock downloadedBlock;
ChainedBlock nextBlock = context.DownloadStack.Peek();
if (context.BlockStoreLoop.BlockPuller.TryGetBlock(nextBlock, out downloadedBlock))
{
this.logger.LogTrace("Puller provided block '{0}', length {1}.", nextBlock, downloadedBlock.Length);
ChainedBlock lastBlockToPush = this.AddDownloadedBlockToStore(context, downloadedBlock);
if (this.ShouldBlocksBePushedToRepository(context))
{
await this.PushBlocksToRepositoryAsync(context, lastBlockToPush);
if (!context.DownloadStack.Any())
{
this.logger.LogTrace("(-)[EMPTY_STACK2]:{0}", InnerStepResult.Stop);
return(InnerStepResult.Stop);
}
}
}
else
{
if (context.StallCount > context.StallCountThreshold)
{
// Increase limit by 10 % to allow adjustments for low speed connections.
// Eventually, the limit be high enough to allow normal operation.
context.StallCountThreshold += context.StallCountThreshold / 10;
this.logger.LogTrace("Stall count threshold increased to {0}.", context.StallCountThreshold);
this.logger.LogTrace("(-)[STALLING]:{0}", InnerStepResult.Stop);
return(InnerStepResult.Stop);
}
this.logger.LogTrace("Block '{0}' not available, stall count is {1}, waiting {2} ms...", nextBlock, context.StallCount, BlockStoreInnerStepContext.StallDelayMs);
await Task.Delay(BlockStoreInnerStepContext.StallDelayMs, context.CancellationToken);
context.StallCount++;
}
}
context.BlocksPushedCount = 0;
this.logger.LogTrace("(-):{0}", InnerStepResult.Next);
return(InnerStepResult.Next);
}
/// <summary> Adds the downloaded block to the store and resets the stall count.</summary>
private ChainedBlock AddDownloadedBlockToStore(BlockStoreInnerStepContext context, DownloadedBlock downloadedBlock)
{
ChainedBlock chainedBlockToStore = context.DownloadStack.Dequeue();
context.Store.Add(new BlockPair(downloadedBlock.Block, chainedBlockToStore));
context.InsertBlockSize += downloadedBlock.Length;
context.StallCount = 0;
this.logger.LogTrace("{0}='{1}/{2}' added to the store.", nameof(chainedBlockToStore), chainedBlockToStore.HashBlock, chainedBlockToStore.Height, context.BlocksPushedCount);
return(chainedBlockToStore);
}
/// <summary>
/// Push (persist) the downloaded blocks to the block repository
/// </summary>
/// <param name="lastDownloadedBlock">Last block in the list to store, also used to set the store tip.</param>
private async Task PushBlocksToRepository(BlockStoreInnerStepContext context, ChainedBlock lastDownloadedBlock)
{
var blocksToStore = context.Store.Select(bp => bp.Block).ToList();
await context.BlockStoreLoop.BlockRepository.PutAsync(lastDownloadedBlock.HashBlock, blocksToStore);
context.BlocksPushedCount += blocksToStore.Count;
this.logger.LogTrace("{0} blocks pushed to the repository. {1}={2}", blocksToStore.Count, nameof(context.BlocksPushedCount), context.BlocksPushedCount);
context.BlockStoreLoop.SetStoreTip(lastDownloadedBlock);
context.InsertBlockSize = 0;
context.LastDownloadStackFlushTime = context.DateTimeProvider.GetUtcNow();
context.Store.Clear();
}
/// <summary> Determines whether or not its time for <see cref="BlockStoreInnerStepReadBlocks"/>
/// to push (persist) the downloaded blocks to the repository.</summary>
private bool ShouldBlocksBePushedToRepository(BlockStoreInnerStepContext context)
{
var now = context.DateTimeProvider.GetUtcNow();
uint lastFlushDiff = (uint)(now - context.LastDownloadStackFlushTime).TotalMilliseconds;
this.logger.LogTrace("Insert block size is {0} bytes, download stack contains {1} more blocks to download, last flush time was {2} ms ago.", context.InsertBlockSize, context.DownloadStack.Count, lastFlushDiff);
var pushBufferSizeReached = context.InsertBlockSize > BlockStoreLoop.MaxInsertBlockSize;
var downloadStackEmpty = !context.DownloadStack.Any();
var pushTimeReached = lastFlushDiff > BlockStoreInnerStepContext.MaxDownloadStackFlushTimeMs;
this.logger.LogTrace("{0}={1} / {2}={3} / {4}={5}", nameof(pushBufferSizeReached), pushBufferSizeReached, nameof(downloadStackEmpty), downloadStackEmpty, nameof(pushTimeReached), pushTimeReached);
return(pushBufferSizeReached || downloadStackEmpty || pushTimeReached);
}
/// <summary> Adds the downloaded block to the store and resets the stall count.</summary>
private ChainedBlock AddDownloadedBlockToStore(BlockStoreInnerStepContext context, DownloadedBlock downloadedBlock)
{
this.logger.LogTrace("({0}.{1}:{2})", nameof(downloadedBlock), nameof(downloadedBlock.Length), downloadedBlock.Length);
ChainedBlock chainedBlockToStore = context.DownloadStack.Dequeue();
context.Store.Add(new BlockPair(downloadedBlock.Block, chainedBlockToStore));
context.InsertBlockSize += downloadedBlock.Length;
context.StallCount = 0;
this.logger.LogTrace("(-):'{0}'", chainedBlockToStore);
return(chainedBlockToStore);
}
/// <summary>
/// Push (persist) the downloaded blocks to the block repository
/// </summary>
/// <param name="lastDownloadedHeader>Last block in the list to store, also used to set the store tip.</param>
private async Task PushBlocksToRepositoryAsync(BlockStoreInnerStepContext context, ChainedHeader lastDownloadedHeader)
{
this.logger.LogTrace("()");
List <Block> blocksToStore = context.Store.Select(bp => bp.Block).ToList();
await context.BlockStoreLoop.BlockRepository.PutAsync(lastDownloadedHeader.HashBlock, blocksToStore);
context.BlocksPushedCount += blocksToStore.Count;
this.logger.LogTrace("{0} blocks pushed to the repository, {1} blocks pushed in total.", blocksToStore.Count, context.BlocksPushedCount);
context.BlockStoreLoop.SetStoreTip(lastDownloadedHeader);
context.InsertBlockSize = 0;
context.LastDownloadStackFlushTime = context.DateTimeProvider.GetUtcNow();
context.Store.Clear();
this.logger.LogTrace("(-)");
}
/// <summary> Determines whether or not its time for <see cref="BlockStoreInnerStepReadBlocks"/>
/// to push (persist) the downloaded blocks to the repository.</summary>
private bool ShouldBlocksBePushedToRepository(BlockStoreInnerStepContext context)
{
this.logger.LogTrace("()");
DateTime now = context.DateTimeProvider.GetUtcNow();
uint lastFlushDiff = (uint)(now - context.LastDownloadStackFlushTime).TotalMilliseconds;
bool pushBufferSizeReached = context.InsertBlockSize > BlockStoreLoop.MaxInsertBlockSize;
bool downloadStackEmpty = !context.DownloadStack.Any();
bool pushTimeReached = lastFlushDiff > BlockStoreInnerStepContext.MaxDownloadStackFlushTimeMs;
this.logger.LogTrace("Insert block size is {0} bytes{1}, download stack contains {2} blocks, last flush time was {3} ms ago{4}.",
context.InsertBlockSize, pushBufferSizeReached ? " (threshold reached)" : "", context.DownloadStack.Count, lastFlushDiff, pushTimeReached ? " (threshold reached)" : "");
bool res = pushBufferSizeReached || downloadStackEmpty || pushTimeReached;
this.logger.LogTrace("(-):{0}", res);
return(res);
}
/// <inheritdoc/>
public override async Task <InnerStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
{
this.logger.LogTrace("()");
while (await ShouldStopFindingBlocks(context) == false)
{
context.DownloadStack.Enqueue(context.NextChainedBlock);
context.GetNextBlock();
}
context.StopFindingBlocks();
if (context.DownloadStack.Any())
{
context.BlockStoreLoop.BlockPuller.AskForMultipleBlocks(context.DownloadStack.ToArray());
}
this.logger.LogTrace("(-):{0}", InnerStepResult.Next);
return(InnerStepResult.Next);
}
/// <inheritdoc/>
public override async Task <BlockStoreLoopStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
{
BlockPuller.DownloadedBlock downloadedBlock;
if (context.BlockStoreLoop.BlockPuller.TryGetBlock(context.DownloadStack.Peek(), out downloadedBlock))
{
var chainedBlockToDownload = context.DownloadStack.Dequeue();
context.Store.Add(new BlockPair(downloadedBlock.Block, chainedBlockToDownload));
context.InsertBlockSize += downloadedBlock.Length;
context.StallCount = 0;
if (context.InsertBlockSize > context.BlockStoreLoop.InsertBlockSizeThreshold || !context.DownloadStack.Any())
{
var blocksToStore = context.Store.Select(bp => bp.Block).ToList();
await context.BlockStoreLoop.BlockRepository.PutAsync(chainedBlockToDownload.HashBlock, blocksToStore);
context.BlockStoreLoop.SetStoreTip(chainedBlockToDownload);
context.InsertBlockSize = 0;
context.Store.Clear();
if (!context.DownloadStack.Any())
{
return(BlockStoreLoopStepResult.Break());
}
}
}
else
{
if (context.StallCount > 10000)
{
return(BlockStoreLoopStepResult.Break());
}
await Task.Delay(100, context.CancellationToken);
context.StallCount++;
}
return(BlockStoreLoopStepResult.Next());
}
private async Task <bool> ShouldStopFindingBlocksAsync(BlockStoreInnerStepContext context)
{
this.logger.LogTrace("()");
if (context.NextChainedHeader == null)
{
this.logger.LogTrace("(-)[NULL_NEXT]:true");
return(true);
}
if ((context.InputChainedHeader != null) && (context.NextChainedHeader.Header.HashPrevBlock != context.InputChainedHeader.HashBlock))
{
this.logger.LogTrace("(-)[NEXT_NEQ_INPUT]:true");
return(true);
}
if (context.NextChainedHeader.Height > context.BlockStoreLoop.ChainState.ConsensusTip?.Height)
{
this.logger.LogTrace("(-)[NEXT_HEIGHT_GT_CONSENSUS_TIP]:true");
return(true);
}
if (context.BlockStoreLoop.PendingStorage.ContainsKey(context.NextChainedHeader.HashBlock))
{
this.logger.LogTrace("Chained block '{0}' already exists in the pending storage.", context.NextChainedHeader);
this.logger.LogTrace("(-)[NEXT_ALREADY_EXISTS_PENDING_STORE]:true");
return(true);
}
if (await context.BlockStoreLoop.BlockRepository.ExistAsync(context.NextChainedHeader.HashBlock))
{
this.logger.LogTrace("Chained block '{0}' already exists in the repository.", context.NextChainedHeader);
this.logger.LogTrace("(-)[NEXT_ALREADY_EXISTS_REPOSITORY]:true");
return(true);
}
this.logger.LogTrace("(-):false");
return(false);
}
/// <inheritdoc/>
public override async Task <InnerStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
{
this.logger.LogTrace("()");
if (!context.DownloadStack.Any())
{
this.logger.LogTrace("(-):{0}", InnerStepResult.Stop);
return(InnerStepResult.Stop);
}
BlockPuller.DownloadedBlock downloadedBlock;
ChainedBlock nextBlock = context.DownloadStack.Peek();
if (context.BlockStoreLoop.BlockPuller.TryGetBlock(nextBlock, out downloadedBlock))
{
this.logger.LogTrace("Puller provided block '{0}/{1}', length {2}.", nextBlock.HashBlock, nextBlock.Height, downloadedBlock.Length);
ChainedBlock chainedBlockToDownload = context.DownloadStack.Dequeue();
context.Store.Add(new BlockPair(downloadedBlock.Block, chainedBlockToDownload));
context.InsertBlockSize += downloadedBlock.Length;
context.StallCount = 0;
DateTime now = context.DateTimeProvider.GetUtcNow();
uint lastFlushDiff = (uint)(now - context.LastDownloadStackFlushTime).TotalMilliseconds;
this.logger.LogTrace("Insert block size is {0} bytes, download stack contains {1} more blocks to download, last flush time was {2} ms ago.", context.InsertBlockSize, context.DownloadStack.Count, lastFlushDiff);
bool flushBufferSizeReached = context.InsertBlockSize > BlockStoreLoop.MaxInsertBlockSize;
bool downloadStackEmpty = !context.DownloadStack.Any();
bool flushTimeReached = lastFlushDiff > BlockStoreInnerStepContext.MaxDownloadStackFlushTimeMs;
if (flushBufferSizeReached || flushTimeReached || downloadStackEmpty)
{
List <Block> blocksToStore = context.Store.Select(bp => bp.Block).ToList();
await context.BlockStoreLoop.BlockRepository.PutAsync(chainedBlockToDownload.HashBlock, blocksToStore);
context.BlockStoreLoop.SetStoreTip(chainedBlockToDownload);
context.InsertBlockSize = 0;
context.Store.Clear();
context.LastDownloadStackFlushTime = context.DateTimeProvider.GetUtcNow();
if (!context.DownloadStack.Any())
{
this.logger.LogTrace("(-):{0}", InnerStepResult.Stop);
return(InnerStepResult.Stop);
}
}
}
else
{
if (context.StallCount > context.StallCountThreshold)
{
// Increase limit by 10 % to allow adjustments for low speed connections.
// Eventually, the limit be high enough to allow normal operation.
context.StallCountThreshold += context.StallCountThreshold / 10;
this.logger.LogTrace("(-):{0},{1}={2}", InnerStepResult.Stop, nameof(context.StallCountThreshold), context.StallCountThreshold);
return(InnerStepResult.Stop);
}
this.logger.LogTrace("Block '{0}/{1}' not available, stall count is {2}, waiting {3} ms...", nextBlock.HashBlock, nextBlock.Height, context.StallCount, BlockStoreInnerStepContext.StallDelayMs);
await Task.Delay(BlockStoreInnerStepContext.StallDelayMs, context.CancellationToken);
context.StallCount++;
}
this.logger.LogTrace("(-):{0}", InnerStepResult.Next);
return(InnerStepResult.Next);
}
public abstract Task <InnerStepResult> ExecuteAsync(BlockStoreInnerStepContext context);
