Exemple #1
0
        /// <inheritdoc/>
        public override async Task <InnerStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
        {
            var batchSize = BlockStoreInnerStepContext.DownloadStackThreshold - context.DownloadStack.Count;
            var batchList = new List <ChainedHeader>(batchSize);

            while (batchList.Count < batchSize)
            {
                if (await this.ShouldStopFindingBlocksAsync(context))
                {
                    context.StopFindingBlocks();
                    break;
                }

                batchList.Add(context.NextChainedHeader);
                context.DownloadStack.Enqueue(context.NextChainedHeader);
                context.GetNextBlock();
            }

            if (batchList.Any())
            {
                this.logger.LogTrace("{0} blocks requested to be downloaded by the puller.", batchList.Count);
                context.BlockStoreLoop.BlockPuller.AskForMultipleBlocks(batchList.ToArray());
            }

            return(InnerStepResult.Next);
        }
Exemple #2
0
        /// <inheritdoc/>
        internal override async Task <BlockStoreLoopStepResult> ExecuteAsync(ChainedBlock nextChainedBlock, CancellationToken token, bool disposeMode)
        {
            if (disposeMode)
            {
                return(BlockStoreLoopStepResult.Break());
            }

            var result = BlockStoreLoopStepResult.Next();

            var context = new BlockStoreInnerStepContext(token, this.BlockStoreLoop).Initialize(nextChainedBlock);

            this.BlockStoreLoop.BlockPuller.AskBlock(nextChainedBlock);

            while (!token.IsCancellationRequested)
            {
                foreach (var item in context.Routine.ToList())
                {
                    var executionResult = await item.ExecuteAsync(context);

                    if (executionResult.ShouldBreak)
                    {
                        return(result);
                    }
                }
            }

            return(result);
        }
Exemple #3
0
        private async Task <bool> ShouldStopFindingBlocks(BlockStoreInnerStepContext context)
        {
            if (context.NextChainedBlock == null)
            {
                return(true);
            }

            if (context.DownloadStack.Count >= BlockStoreInnerStepContext.DownloadStackThreshold)
            {
                return(true);
            }

            if (context.InputChainedBlock != null && (context.NextChainedBlock.Header.HashPrevBlock != context.InputChainedBlock.HashBlock))
            {
                return(true);
            }

            if (context.NextChainedBlock.Height > context.BlockStoreLoop.ChainState.HighestValidatedPoW?.Height)
            {
                return(true);
            }

            if (context.BlockStoreLoop.PendingStorage.ContainsKey(context.NextChainedBlock.HashBlock))
            {
                return(true);
            }

            if (await context.BlockStoreLoop.BlockRepository.ExistAsync(context.NextChainedBlock.HashBlock))
            {
                return(true);
            }

            return(false);
        }
        /// <inheritdoc/>
        public override async Task <BlockStoreLoopStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
        {
            context.GetNextBlock();

            if (await ShouldStopFindingBlocks(context))
            {
                if (!context.DownloadStack.Any())
                {
                    return(BlockStoreLoopStepResult.Break());
                }

                context.StopFindingBlocks();
            }
            else
            {
                context.BlockStoreLoop.BlockPuller.AskBlock(context.NextChainedBlock);
                context.DownloadStack.Enqueue(context.NextChainedBlock);

                if (context.DownloadStack.Count == context.BlockStoreLoop.BatchDownloadSize)
                {
                    context.StopFindingBlocks();
                }
            }

            return(BlockStoreLoopStepResult.Next());
        }
Exemple #5
0
        /// <inheritdoc/>
        internal override async Task <StepResult> ExecuteAsync(ChainedBlock nextChainedBlock, CancellationToken token, bool disposeMode)
        {
            this.logger.LogTrace("({0}:'{1}/{2}',{3}:{4})", nameof(nextChainedBlock), nextChainedBlock?.HashBlock, nextChainedBlock?.Height, nameof(disposeMode), disposeMode);

            if (disposeMode)
            {
                this.logger.LogTrace("(-):{0}", StepResult.Stop);
                return(StepResult.Stop);
            }

            var context = new BlockStoreInnerStepContext(token, this.BlockStoreLoop, nextChainedBlock, this.loggerFactory, this.dateTimeProvider);

            while (!token.IsCancellationRequested)
            {
                foreach (var innerStep in context.InnerSteps.ToList())
                {
                    InnerStepResult innerStepResult = await innerStep.ExecuteAsync(context);

                    if (innerStepResult == InnerStepResult.Stop)
                    {
                        this.logger.LogTrace("(-):{0}", StepResult.Next);
                        return(StepResult.Next);
                    }
                }
            }

            this.logger.LogTrace("(-):{0}", StepResult.Next);
            return(StepResult.Next);
        }
        private async Task <bool> ShouldStopFindingBlocks(BlockStoreInnerStepContext context)
        {
            if (context.NextChainedBlock == null)
            {
                this.logger.LogTrace("{0} is null", nameof(context.NextChainedBlock));
                return(true);
            }

            if (context.InputChainedBlock != null && (context.NextChainedBlock.Header.HashPrevBlock != context.InputChainedBlock.HashBlock))
            {
                this.logger.LogTrace("{0} != {1}", nameof(context.NextChainedBlock.Header.HashPrevBlock), nameof(context.InputChainedBlock.HashBlock));
                return(true);
            }

            if (context.NextChainedBlock.Height > context.BlockStoreLoop.ChainState.HighestValidatedPoW?.Height)
            {
                this.logger.LogTrace("{0} height > {1} height", nameof(context.NextChainedBlock), nameof(context.BlockStoreLoop.ChainState.HighestValidatedPoW));
                return(true);
            }

            if (context.BlockStoreLoop.PendingStorage.ContainsKey(context.NextChainedBlock.HashBlock))
            {
                this.logger.LogTrace("{0}='{1}/{2}' exists in pending storage.", nameof(context.NextChainedBlock), context.NextChainedBlock.HashBlock, context.NextChainedBlock.Height);
                return(true);
            }

            if (await context.BlockStoreLoop.BlockRepository.ExistAsync(context.NextChainedBlock.HashBlock))
            {
                this.logger.LogTrace("{0}='{1}/{2}' exists in the repository.", nameof(context.NextChainedBlock), context.NextChainedBlock.HashBlock, context.NextChainedBlock.Height);
                return(true);
            }

            return(false);
        }
Exemple #7
0
        /// <inheritdoc/>
        public override async Task <InnerStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
        {
            if (!context.DownloadStack.Any())
            {
                this.logger.LogTrace("(-)[EMPTY_STACK1]:{0}", InnerStepResult.Stop);
                return(InnerStepResult.Stop);
            }

            while (context.BlocksPushedCount <= BlockStoreInnerStepContext.DownloadStackPushThreshold)
            {
                DownloadedBlock downloadedBlock;
                ChainedBlock    nextBlock = context.DownloadStack.Peek();

                if (context.BlockStoreLoop.BlockPuller.TryGetBlock(nextBlock, out downloadedBlock))
                {
                    this.logger.LogTrace("Puller provided block '{0}', length {1}.", nextBlock, downloadedBlock.Length);

                    ChainedBlock lastBlockToPush = this.AddDownloadedBlockToStore(context, downloadedBlock);

                    if (this.ShouldBlocksBePushedToRepository(context))
                    {
                        await this.PushBlocksToRepositoryAsync(context, lastBlockToPush);

                        if (!context.DownloadStack.Any())
                        {
                            this.logger.LogTrace("(-)[EMPTY_STACK2]:{0}", InnerStepResult.Stop);
                            return(InnerStepResult.Stop);
                        }
                    }
                }
                else
                {
                    if (context.StallCount > context.StallCountThreshold)
                    {
                        // Increase limit by 10 % to allow adjustments for low speed connections.
                        // Eventually, the limit be high enough to allow normal operation.
                        context.StallCountThreshold += context.StallCountThreshold / 10;
                        this.logger.LogTrace("Stall count threshold increased to {0}.", context.StallCountThreshold);
                        this.logger.LogTrace("(-)[STALLING]:{0}", InnerStepResult.Stop);
                        return(InnerStepResult.Stop);
                    }

                    this.logger.LogTrace("Block '{0}' not available, stall count is {1}, waiting {2} ms...", nextBlock, context.StallCount, BlockStoreInnerStepContext.StallDelayMs);
                    await Task.Delay(BlockStoreInnerStepContext.StallDelayMs, context.CancellationToken);

                    context.StallCount++;
                }
            }

            context.BlocksPushedCount = 0;

            this.logger.LogTrace("(-):{0}", InnerStepResult.Next);
            return(InnerStepResult.Next);
        }
        /// <summary> Adds the downloaded block to the store and resets the stall count.</summary>
        private ChainedBlock AddDownloadedBlockToStore(BlockStoreInnerStepContext context, DownloadedBlock downloadedBlock)
        {
            ChainedBlock chainedBlockToStore = context.DownloadStack.Dequeue();

            context.Store.Add(new BlockPair(downloadedBlock.Block, chainedBlockToStore));

            context.InsertBlockSize += downloadedBlock.Length;
            context.StallCount       = 0;

            this.logger.LogTrace("{0}='{1}/{2}' added to the store.", nameof(chainedBlockToStore), chainedBlockToStore.HashBlock, chainedBlockToStore.Height, context.BlocksPushedCount);
            return(chainedBlockToStore);
        }
        /// <summary>
        /// Push (persist) the downloaded blocks to the block repository
        /// </summary>
        /// <param name="lastDownloadedBlock">Last block in the list to store, also used to set the store tip.</param>
        private async Task PushBlocksToRepository(BlockStoreInnerStepContext context, ChainedBlock lastDownloadedBlock)
        {
            var blocksToStore = context.Store.Select(bp => bp.Block).ToList();
            await context.BlockStoreLoop.BlockRepository.PutAsync(lastDownloadedBlock.HashBlock, blocksToStore);

            context.BlocksPushedCount += blocksToStore.Count;
            this.logger.LogTrace("{0} blocks pushed to the repository. {1}={2}", blocksToStore.Count, nameof(context.BlocksPushedCount), context.BlocksPushedCount);

            context.BlockStoreLoop.SetStoreTip(lastDownloadedBlock);
            context.InsertBlockSize            = 0;
            context.LastDownloadStackFlushTime = context.DateTimeProvider.GetUtcNow();

            context.Store.Clear();
        }
        /// <summary> Determines whether or not its time for <see cref="BlockStoreInnerStepReadBlocks"/>
        /// to push (persist) the downloaded blocks to the repository.</summary>
        private bool ShouldBlocksBePushedToRepository(BlockStoreInnerStepContext context)
        {
            var  now           = context.DateTimeProvider.GetUtcNow();
            uint lastFlushDiff = (uint)(now - context.LastDownloadStackFlushTime).TotalMilliseconds;

            this.logger.LogTrace("Insert block size is {0} bytes, download stack contains {1} more blocks to download, last flush time was {2} ms ago.", context.InsertBlockSize, context.DownloadStack.Count, lastFlushDiff);

            var pushBufferSizeReached = context.InsertBlockSize > BlockStoreLoop.MaxInsertBlockSize;
            var downloadStackEmpty    = !context.DownloadStack.Any();
            var pushTimeReached       = lastFlushDiff > BlockStoreInnerStepContext.MaxDownloadStackFlushTimeMs;

            this.logger.LogTrace("{0}={1} / {2}={3} / {4}={5}", nameof(pushBufferSizeReached), pushBufferSizeReached, nameof(downloadStackEmpty), downloadStackEmpty, nameof(pushTimeReached), pushTimeReached);
            return(pushBufferSizeReached || downloadStackEmpty || pushTimeReached);
        }
Exemple #11
0
        /// <summary> Adds the downloaded block to the store and resets the stall count.</summary>
        private ChainedBlock AddDownloadedBlockToStore(BlockStoreInnerStepContext context, DownloadedBlock downloadedBlock)
        {
            this.logger.LogTrace("({0}.{1}:{2})", nameof(downloadedBlock), nameof(downloadedBlock.Length), downloadedBlock.Length);

            ChainedBlock chainedBlockToStore = context.DownloadStack.Dequeue();

            context.Store.Add(new BlockPair(downloadedBlock.Block, chainedBlockToStore));

            context.InsertBlockSize += downloadedBlock.Length;
            context.StallCount       = 0;

            this.logger.LogTrace("(-):'{0}'", chainedBlockToStore);
            return(chainedBlockToStore);
        }
        /// <summary>
        /// Push (persist) the downloaded blocks to the block repository
        /// </summary>
        /// <param name="lastDownloadedHeader>Last block in the list to store, also used to set the store tip.</param>
        private async Task PushBlocksToRepositoryAsync(BlockStoreInnerStepContext context, ChainedHeader lastDownloadedHeader)
        {
            this.logger.LogTrace("()");

            List <Block> blocksToStore = context.Store.Select(bp => bp.Block).ToList();
            await context.BlockStoreLoop.BlockRepository.PutAsync(lastDownloadedHeader.HashBlock, blocksToStore);

            context.BlocksPushedCount += blocksToStore.Count;
            this.logger.LogTrace("{0} blocks pushed to the repository, {1} blocks pushed in total.", blocksToStore.Count, context.BlocksPushedCount);

            context.BlockStoreLoop.SetStoreTip(lastDownloadedHeader);
            context.InsertBlockSize            = 0;
            context.LastDownloadStackFlushTime = context.DateTimeProvider.GetUtcNow();

            context.Store.Clear();

            this.logger.LogTrace("(-)");
        }
Exemple #13
0
        /// <summary> Determines whether or not its time for <see cref="BlockStoreInnerStepReadBlocks"/>
        /// to push (persist) the downloaded blocks to the repository.</summary>
        private bool ShouldBlocksBePushedToRepository(BlockStoreInnerStepContext context)
        {
            this.logger.LogTrace("()");

            DateTime now           = context.DateTimeProvider.GetUtcNow();
            uint     lastFlushDiff = (uint)(now - context.LastDownloadStackFlushTime).TotalMilliseconds;

            bool pushBufferSizeReached = context.InsertBlockSize > BlockStoreLoop.MaxInsertBlockSize;
            bool downloadStackEmpty    = !context.DownloadStack.Any();
            bool pushTimeReached       = lastFlushDiff > BlockStoreInnerStepContext.MaxDownloadStackFlushTimeMs;

            this.logger.LogTrace("Insert block size is {0} bytes{1}, download stack contains {2} blocks, last flush time was {3} ms ago{4}.",
                                 context.InsertBlockSize, pushBufferSizeReached ? " (threshold reached)" : "", context.DownloadStack.Count, lastFlushDiff, pushTimeReached ? " (threshold reached)" : "");

            bool res = pushBufferSizeReached || downloadStackEmpty || pushTimeReached;

            this.logger.LogTrace("(-):{0}", res);
            return(res);
        }
Exemple #14
0
        /// <inheritdoc/>
        public override async Task <InnerStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
        {
            this.logger.LogTrace("()");

            while (await ShouldStopFindingBlocks(context) == false)
            {
                context.DownloadStack.Enqueue(context.NextChainedBlock);
                context.GetNextBlock();
            }

            context.StopFindingBlocks();

            if (context.DownloadStack.Any())
            {
                context.BlockStoreLoop.BlockPuller.AskForMultipleBlocks(context.DownloadStack.ToArray());
            }

            this.logger.LogTrace("(-):{0}", InnerStepResult.Next);

            return(InnerStepResult.Next);
        }
        /// <inheritdoc/>
        public override async Task <BlockStoreLoopStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
        {
            BlockPuller.DownloadedBlock downloadedBlock;

            if (context.BlockStoreLoop.BlockPuller.TryGetBlock(context.DownloadStack.Peek(), out downloadedBlock))
            {
                var chainedBlockToDownload = context.DownloadStack.Dequeue();
                context.Store.Add(new BlockPair(downloadedBlock.Block, chainedBlockToDownload));
                context.InsertBlockSize += downloadedBlock.Length;
                context.StallCount       = 0;

                if (context.InsertBlockSize > context.BlockStoreLoop.InsertBlockSizeThreshold || !context.DownloadStack.Any())
                {
                    var blocksToStore = context.Store.Select(bp => bp.Block).ToList();
                    await context.BlockStoreLoop.BlockRepository.PutAsync(chainedBlockToDownload.HashBlock, blocksToStore);

                    context.BlockStoreLoop.SetStoreTip(chainedBlockToDownload);
                    context.InsertBlockSize = 0;
                    context.Store.Clear();

                    if (!context.DownloadStack.Any())
                    {
                        return(BlockStoreLoopStepResult.Break());
                    }
                }
            }
            else
            {
                if (context.StallCount > 10000)
                {
                    return(BlockStoreLoopStepResult.Break());
                }

                await Task.Delay(100, context.CancellationToken);

                context.StallCount++;
            }

            return(BlockStoreLoopStepResult.Next());
        }
Exemple #16
0
        private async Task <bool> ShouldStopFindingBlocksAsync(BlockStoreInnerStepContext context)
        {
            this.logger.LogTrace("()");

            if (context.NextChainedHeader == null)
            {
                this.logger.LogTrace("(-)[NULL_NEXT]:true");
                return(true);
            }

            if ((context.InputChainedHeader != null) && (context.NextChainedHeader.Header.HashPrevBlock != context.InputChainedHeader.HashBlock))
            {
                this.logger.LogTrace("(-)[NEXT_NEQ_INPUT]:true");
                return(true);
            }

            if (context.NextChainedHeader.Height > context.BlockStoreLoop.ChainState.ConsensusTip?.Height)
            {
                this.logger.LogTrace("(-)[NEXT_HEIGHT_GT_CONSENSUS_TIP]:true");
                return(true);
            }

            if (context.BlockStoreLoop.PendingStorage.ContainsKey(context.NextChainedHeader.HashBlock))
            {
                this.logger.LogTrace("Chained block '{0}' already exists in the pending storage.", context.NextChainedHeader);
                this.logger.LogTrace("(-)[NEXT_ALREADY_EXISTS_PENDING_STORE]:true");
                return(true);
            }

            if (await context.BlockStoreLoop.BlockRepository.ExistAsync(context.NextChainedHeader.HashBlock))
            {
                this.logger.LogTrace("Chained block '{0}' already exists in the repository.", context.NextChainedHeader);
                this.logger.LogTrace("(-)[NEXT_ALREADY_EXISTS_REPOSITORY]:true");
                return(true);
            }

            this.logger.LogTrace("(-):false");
            return(false);
        }
Exemple #17
0
        /// <inheritdoc/>
        public override async Task <InnerStepResult> ExecuteAsync(BlockStoreInnerStepContext context)
        {
            this.logger.LogTrace("()");

            if (!context.DownloadStack.Any())
            {
                this.logger.LogTrace("(-):{0}", InnerStepResult.Stop);
                return(InnerStepResult.Stop);
            }

            BlockPuller.DownloadedBlock downloadedBlock;

            ChainedBlock nextBlock = context.DownloadStack.Peek();

            if (context.BlockStoreLoop.BlockPuller.TryGetBlock(nextBlock, out downloadedBlock))
            {
                this.logger.LogTrace("Puller provided block '{0}/{1}', length {2}.", nextBlock.HashBlock, nextBlock.Height, downloadedBlock.Length);

                ChainedBlock chainedBlockToDownload = context.DownloadStack.Dequeue();
                context.Store.Add(new BlockPair(downloadedBlock.Block, chainedBlockToDownload));
                context.InsertBlockSize += downloadedBlock.Length;
                context.StallCount       = 0;

                DateTime now           = context.DateTimeProvider.GetUtcNow();
                uint     lastFlushDiff = (uint)(now - context.LastDownloadStackFlushTime).TotalMilliseconds;
                this.logger.LogTrace("Insert block size is {0} bytes, download stack contains {1} more blocks to download, last flush time was {2} ms ago.", context.InsertBlockSize, context.DownloadStack.Count, lastFlushDiff);

                bool flushBufferSizeReached = context.InsertBlockSize > BlockStoreLoop.MaxInsertBlockSize;
                bool downloadStackEmpty     = !context.DownloadStack.Any();
                bool flushTimeReached       = lastFlushDiff > BlockStoreInnerStepContext.MaxDownloadStackFlushTimeMs;

                if (flushBufferSizeReached || flushTimeReached || downloadStackEmpty)
                {
                    List <Block> blocksToStore = context.Store.Select(bp => bp.Block).ToList();
                    await context.BlockStoreLoop.BlockRepository.PutAsync(chainedBlockToDownload.HashBlock, blocksToStore);

                    context.BlockStoreLoop.SetStoreTip(chainedBlockToDownload);
                    context.InsertBlockSize = 0;
                    context.Store.Clear();
                    context.LastDownloadStackFlushTime = context.DateTimeProvider.GetUtcNow();

                    if (!context.DownloadStack.Any())
                    {
                        this.logger.LogTrace("(-):{0}", InnerStepResult.Stop);
                        return(InnerStepResult.Stop);
                    }
                }
            }
            else
            {
                if (context.StallCount > context.StallCountThreshold)
                {
                    // Increase limit by 10 % to allow adjustments for low speed connections.
                    // Eventually, the limit be high enough to allow normal operation.
                    context.StallCountThreshold += context.StallCountThreshold / 10;
                    this.logger.LogTrace("(-):{0},{1}={2}", InnerStepResult.Stop, nameof(context.StallCountThreshold), context.StallCountThreshold);
                    return(InnerStepResult.Stop);
                }

                this.logger.LogTrace("Block '{0}/{1}' not available, stall count is {2}, waiting {3} ms...", nextBlock.HashBlock, nextBlock.Height, context.StallCount, BlockStoreInnerStepContext.StallDelayMs);
                await Task.Delay(BlockStoreInnerStepContext.StallDelayMs, context.CancellationToken);

                context.StallCount++;
            }

            this.logger.LogTrace("(-):{0}", InnerStepResult.Next);
            return(InnerStepResult.Next);
        }
 public abstract Task <InnerStepResult> ExecuteAsync(BlockStoreInnerStepContext context);