コード例 #1
0
                public Block[] Process(Keccak newBranchStateRoot, List <Block> suggestedBlocks, ProcessingOptions processingOptions, IBlockTracer blockTracer)
                {
                    _logger.Info($"Processing {suggestedBlocks.Last().ToString(Block.Format.Short)}");
                    while (true)
                    {
                        bool notYet = false;
                        for (int i = 0; i < suggestedBlocks.Count; i++)
                        {
                            Keccak hash = suggestedBlocks[i].Hash;
                            if (!_allowed.Contains(hash))
                            {
                                if (_allowedToFail.Contains(hash))
                                {
                                    _allowedToFail.Remove(hash);
                                    BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(suggestedBlocks.Last()));
                                    throw new InvalidBlockException(hash);
                                }

                                notYet = true;
                                break;
                            }
                        }

                        if (notYet)
                        {
                            Thread.Sleep(20);
                        }
                        else
                        {
                            BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(suggestedBlocks.Last()));
                            return(suggestedBlocks.ToArray());
                        }
                    }
                }
コード例 #2
0
ファイル: BlockProcessor.cs プロジェクト: fosfuan/nethermind
        private Block ProcessOne(Block suggestedBlock, ProcessingOptions options, IBlockTracer blockTracer)
        {
            Block block;

            if (suggestedBlock.IsGenesis)
            {
                block = suggestedBlock;
            }
            else
            {
                if (_specProvider.DaoBlockNumber.HasValue && _specProvider.DaoBlockNumber.Value == suggestedBlock.Header.Number)
                {
                    if (_logger.IsInfo)
                    {
                        _logger.Info("Applying DAO transition");
                    }
                    ApplyDaoTransition();
                }

                block = PrepareBlockForProcessing(suggestedBlock);
                _additionalBlockProcessor?.PreProcess(block, options);

                var receipts = ProcessTransactions(block, options, blockTracer);
                SetReceiptsRoot(block, receipts);
                ApplyMinerRewards(block, blockTracer);

                _stateProvider.Commit(_specProvider.GetSpec(block.Number));
                block.Header.StateRoot = _stateProvider.StateRoot;
                block.Header.Hash      = BlockHeader.CalculateHash(block.Header);

                _additionalBlockProcessor?.PostProcess(block, receipts, options);

                if ((options & ProcessingOptions.NoValidation) == 0 && !_blockValidator.ValidateProcessedBlock(block, receipts, suggestedBlock))
                {
                    if (_logger.IsError)
                    {
                        _logger.Error($"Processed block is not valid {suggestedBlock.ToString(Block.Format.FullHashAndNumber)}");
                    }
                    // if (_logger.IsError) _logger.Error($"State: {_stateProvider.DumpState()}");
                    throw new InvalidBlockException(suggestedBlock.Hash);
                }

                if ((options & ProcessingOptions.StoreReceipts) != 0)
                {
                    StoreTxReceipts(block, receipts);
                }

                if ((options & ProcessingOptions.StoreTraces) != 0)
                {
                    StoreTraces(blockTracer as ParityLikeBlockTracer);
                }
            }

            if ((options & ProcessingOptions.ReadOnlyChain) == 0)
            {
                BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(block));
            }

            return(block);
        }
コード例 #3
0
ファイル: Decompression.cs プロジェクト: van8tka/GZipTest
 protected override void ProccessingData(object indexThread)
 {
     try
     {
         byte[] hederBytes;
         int    lenghtBlock;
         int    dataLenght;
         byte[] data;
         long   position;
         long   sizefile;
         while (true && !IsError)
         {
             BlockData block;
             if (BlockReaded.TryTakeBlock(out block))
             {
                 using (var memStream = new MemoryStream(block.Bytes))
                 {
                     using (var gzipStream = new GZipStream(memStream, CompressionMode.Decompress))
                     {
                         hederBytes = new byte[4] {
                             block.Bytes[4], block.Bytes[5], block.Bytes[6], block.Bytes[7]
                         };
                         lenghtBlock = BitConverter.ToInt32(hederBytes, 0);
                         dataLenght  = BitConverter.ToInt32(block.Bytes, lenghtBlock - 4);
                         data        = new byte[dataLenght];
                         gzipStream.Read(data, 0, dataLenght);
                         //отделяем от data первые 8 байт - 4 байта позиция записи, 4 байта размер файла
                         data = GetHelpersData(out position, out sizefile, data);
                         BlockProcessed.AddBlock(new BlockData(position, sizefile, data));
                         BlocksProcessedCount++;
                     }
                 }
             }
             else
             {
                 if (BlockReaded.IsFinish && BlocksProcessedCount == BlocksCount)
                 {
                     BlockProcessed.Finish();
                 }
                 return;
             }
         }
     }
     catch (IOException e)
     {
         ErrorOutput(e, "Unable to complete decompression operation due to insufficient RAM. Please close other applications and try again. ");
     }
     catch (OutOfMemoryException e)
     {
         ErrorOutput(e, "Not enough RAM to complete file decompression. Please close other applications and try again. ");
     }
     catch (Exception e)
     {
         ErrorOutput(e);
     }
     finally
     {
         EventWaitHandleArray[(int)indexThread].Set();
     }
 }
コード例 #4
0
        // TODO: move to branch processor
        public Block[] Process(Keccak newBranchStateRoot, List <Block> suggestedBlocks, ProcessingOptions options, IBlockTracer blockTracer)
        {
            if (suggestedBlocks.Count == 0)
            {
                return(Array.Empty <Block>());
            }

            /* We need to save the snapshot state root before reorganization in case the new branch has invalid blocks.
             * In case of invalid blocks on the new branch we will discard the entire branch and come back to
             * the previous head state.*/
            Keccak previousBranchStateRoot = CreateCheckpoint();

            InitBranch(newBranchStateRoot);

            bool readOnly = (options & ProcessingOptions.ReadOnlyChain) != 0;

            Block[] processedBlocks = new Block[suggestedBlocks.Count];
            try
            {
                for (int i = 0; i < suggestedBlocks.Count; i++)
                {
                    if (suggestedBlocks.Count > 64 && i % 8 == 0)
                    {
                        if (_logger.IsInfo)
                        {
                            _logger.Info($"Processing part of a long blocks branch {i}/{suggestedBlocks.Count}");
                        }
                    }

                    var(processedBlock, receipts) = ProcessOne(suggestedBlocks[i], options, blockTracer);
                    processedBlocks[i]            = processedBlock;

                    // be cautious here as AuRa depends on processing
                    PreCommitBlock(newBranchStateRoot); // only needed if we plan to read state root?
                    if (!readOnly)
                    {
                        BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(processedBlock, receipts));
                    }
                }

                if (readOnly)
                {
                    RestoreBranch(previousBranchStateRoot);
                }
                else
                {
                    // TODO: move to branch processor
                    CommitBranch();
                }

                return(processedBlocks);
            }
            catch (Exception) // try to restore for all cost
            {
                RestoreBranch(previousBranchStateRoot);
                throw;
            }
        }
コード例 #5
0
ファイル: Compression.cs プロジェクト: van8tka/GZipTest
        protected override void WriteData()
        {
            try
            {
                int       blocksWrite = 0;
                BlockData block;
                byte[]    lenghtOfBlock;
                while (true && !IsError)
                {
                    using (var outputStream = new FileStream(OutputFile, FileMode.Append, FileAccess.Write))
                    {
                        if (BlockProcessed.TryTakeBlock(out block))
                        {
                            //получим размер сжатых данных для последующей декомпрессии
                            lenghtOfBlock = BitConverter.GetBytes(block.Bytes.Length);

                            /*запишем информацию о размере блока для последующей декомперссии и записи, вместо
                             * времени модификации файла в формате  MTIME  (согласно спецификации gzip) */
                            lenghtOfBlock.CopyTo(block.Bytes, 4);
                            outputStream.Write(block.Bytes, 0, block.Bytes.Length);
                            blocksWrite++;
                            ProgressInfo.Output(blocksWrite, BlocksCount);
                        }
                        else
                        {
                            if (blocksWrite == BlocksCount)
                            {
                                ProgressInfo.End();
                            }
                            else
                            {
                                throw new Exception("Can't write all blocks");
                            }
                            return;
                        }
                    }
                }
            }
            catch (IOException e)
            {
                ErrorOutput(e, "Unable to complete write operation due to insufficient RAM. Please close other applications and try again. ");
            }
            catch (OutOfMemoryException e)
            {
                ErrorOutput(e, "Not enough RAM to complete file write. Please close other applications and try again. ");
            }
            catch (Exception e)
            {
                ErrorOutput(e);
            }
            finally
            {
                EventWaitHandleWrite.Set();
            }
        }
コード例 #6
0
ファイル: Decompression.cs プロジェクト: van8tka/GZipTest
 protected override void WriteData()
 {
     try
     {
         bool isSetLenght = false;
         int  blocksWrite = 0;
         while (true && !IsError)
         {
             BlockData block;
             if (BlockProcessed.TryTakeBlock(out block))
             {
                 using (var outputStream = new FileStream(OutputFile, FileMode.Open, FileAccess.Write))
                 {
                     if (!isSetLenght)
                     {
                         outputStream.SetLength(block.SizeFile);
                         isSetLenght = true;
                     }
                     outputStream.Seek(block.Position, SeekOrigin.Begin);
                     outputStream.Write(block.Bytes, 0, block.Bytes.Length);
                     blocksWrite++;
                     ProgressInfo.Output(blocksWrite, BlocksCount);
                 }
             }
             else
             {
                 if (blocksWrite >= BlocksCount)
                 {
                     ProgressInfo.End();
                 }
                 else
                 {
                     throw new Exception("Can't write all blocks");
                 }
                 return;
             }
         }
     }
     catch (IOException e)
     {
         ErrorOutput(e, "Unable to complete write operation due to insufficient RAM. Please close other applications and try again. ");
     }
     catch (OutOfMemoryException e)
     {
         ErrorOutput(e, "Not enough RAM to complete file write. Please close other applications and try again. ");
     }
     catch (Exception e)
     {
         ErrorOutput(e);
     }
     finally
     {
         EventWaitHandleWrite.Set();
     }
 }
コード例 #7
0
ファイル: BlockProcessor.cs プロジェクト: AlexSSD7/nethermind
        public Block[] Process(Keccak newBranchStateRoot, List <Block> suggestedBlocks, ProcessingOptions options, IBlockTracer blockTracer)
        {
            if (suggestedBlocks.Count == 0)
            {
                return(Array.Empty <Block>());
            }

            /* We need to save the snapshot state root before reorganization in case the new branch has invalid blocks.
             * In case of invalid blocks on the new branch we will discard the entire branch and come back to
             * the previous head state.*/
            Keccak previousBranchStateRoot = CreateCheckpoint();

            InitBranch(newBranchStateRoot);

            bool readOnly = (options & ProcessingOptions.ReadOnlyChain) != 0;

            Block[] processedBlocks = new Block[suggestedBlocks.Count];
            try
            {
                for (int i = 0; i < suggestedBlocks.Count; i++)
                {
                    processedBlocks[i] = ProcessOne(suggestedBlocks[i], options, blockTracer);

                    // be cautious here as AuRa depends on processing
                    PreCommitBlock(newBranchStateRoot); // only needed if we plan to read state root?
                    if (!readOnly)
                    {
                        BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(processedBlocks[i]));
                    }
                }

                if (readOnly)
                {
                    RestoreBranch(previousBranchStateRoot);
                }
                else
                {
                    CommitBranch();
                }

                return(processedBlocks);
            }
            catch (InvalidBlockException)
            {
                RestoreBranch(previousBranchStateRoot);
                throw;
            }
        }
コード例 #8
0
ファイル: Compression.cs プロジェクト: van8tka/GZipTest
 /// <summary>
 /// сжатие данных
 /// </summary>
 protected override void ProccessingData(object indexThread)
 {
     try
     {
         BlockData block;
         while (true && !IsError)
         {
             if (BlockReaded.TryTakeBlock(out block))
             {
                 using (var memStream = new MemoryStream())
                 {
                     using (var gzipStream = new GZipStream(memStream, CompressionMode.Compress))
                     {
                         gzipStream.Write(block.Bytes, 0, block.Bytes.Length);
                     }
                     BlockProcessed.AddBlock(new BlockData(memStream.ToArray()));
                     BlocksProcessedCount++;
                 }
             }
             else
             {
                 if (BlockReaded.IsFinish && BlocksProcessedCount == BlocksCount)
                 {
                     BlockProcessed.Finish();
                 }
                 return;
             }
         }
     }
     catch (IOException e)
     {
         ErrorOutput(e, "Unable to complete compression operation due to insufficient RAM. Please close other applications and try again. ");
     }
     catch (OutOfMemoryException e)
     {
         ErrorOutput(e, "Not enough RAM to complete file compression. Please close other applications and try again. ");
     }
     catch (Exception e)
     {
         ErrorOutput(e);
     }
     finally
     {
         EventWaitHandleArray[(int)indexThread].Set();
     }
 }
コード例 #9
0
                public Block[] Process(Keccak newBranchStateRoot, List <Block> suggestedBlocks, ProcessingOptions processingOptions, IBlockTracer blockTracer)
                {
                    if (blockTracer != NullBlockTracer.Instance)
                    {
                        // this is for block reruns on failure for diag tracing
                        throw new InvalidBlockException(suggestedBlocks[0]);
                    }

                    _logger.Info($"Processing {suggestedBlocks.Last().ToString(Block.Format.Short)}");
                    while (true)
                    {
                        bool notYet = false;
                        for (int i = 0; i < suggestedBlocks.Count; i++)
                        {
                            BlocksProcessing?.Invoke(this, new BlocksProcessingEventArgs(suggestedBlocks));
                            Block  suggestedBlock = suggestedBlocks[i];
                            Keccak hash           = suggestedBlock.Hash !;
                            if (!_allowed.Contains(hash))
                            {
                                if (_allowedToFail.Contains(hash))
                                {
                                    _allowedToFail.Remove(hash);
                                    BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(suggestedBlocks.Last(), Array.Empty <TxReceipt>()));
                                    throw new InvalidBlockException(suggestedBlock);
                                }

                                notYet = true;
                                break;
                            }
                        }

                        if (notYet)
                        {
                            Thread.Sleep(20);
                        }
                        else
                        {
                            BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(suggestedBlocks.Last(), Array.Empty <TxReceipt>()));
                            return(suggestedBlocks.ToArray());
                        }
                    }
                }
コード例 #10
0
ファイル: BlockProcessor.cs プロジェクト: sounak98/nethermind
        public Block[] Process(Keccak branchStateRoot, Block[] suggestedBlocks, ProcessingOptions options, IBlockTracer blockTracer)
        {
            if (suggestedBlocks.Length == 0)
            {
                return(Array.Empty <Block>());
            }

            int stateSnapshot = _stateDb.TakeSnapshot();
            int codeSnapshot  = _codeDb.TakeSnapshot();

            if (stateSnapshot != -1 || codeSnapshot != -1)
            {
                if (_logger.IsError)
                {
                    _logger.Error($"Uncommitted state ({stateSnapshot}, {codeSnapshot}) when processing from a branch root {branchStateRoot} starting with block {suggestedBlocks[0].ToString(Block.Format.Short)}");
                }
            }

            Keccak snapshotStateRoot = _stateProvider.StateRoot;

            if (branchStateRoot != null && _stateProvider.StateRoot != branchStateRoot)
            {
                /* discarding the other branch data - chain reorganization */
                Metrics.Reorganizations++;
                _storageProvider.Reset();
                _stateProvider.Reset();
                _stateProvider.StateRoot = branchStateRoot;
            }

            var readOnly        = (options & ProcessingOptions.ReadOnlyChain) != 0;
            var processedBlocks = new Block[suggestedBlocks.Length];

            try
            {
                for (int i = 0; i < suggestedBlocks.Length; i++)
                {
                    processedBlocks[i] = ProcessOne(suggestedBlocks[i], options, blockTracer);
                    if (_logger.IsTrace)
                    {
                        _logger.Trace($"Committing trees - state root {_stateProvider.StateRoot}");
                    }
                    _stateProvider.CommitTree();
                    _storageProvider.CommitTrees();

                    if (!readOnly)
                    {
                        BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(processedBlocks[i]));
                    }
                }

                if (readOnly)
                {
                    _receiptsTracer.BeforeRestore(_stateProvider);
                    Restore(stateSnapshot, codeSnapshot, snapshotStateRoot);
                }
                else
                {
                    _stateDb.Commit();
                    _codeDb.Commit();
                }

                return(processedBlocks);
            }
            catch (InvalidBlockException)
            {
                Restore(stateSnapshot, codeSnapshot, snapshotStateRoot);
                throw;
            }
        }
コード例 #11
0
        // TODO: move to branch processor
        public Block[] Process(Keccak newBranchStateRoot, List <Block> suggestedBlocks, ProcessingOptions options, IBlockTracer blockTracer)
        {
            if (suggestedBlocks.Count == 0)
            {
                return(Array.Empty <Block>());
            }

            BlocksProcessing?.Invoke(this, new BlocksProcessingEventArgs(suggestedBlocks));

            /* We need to save the snapshot state root before reorganization in case the new branch has invalid blocks.
             * In case of invalid blocks on the new branch we will discard the entire branch and come back to
             * the previous head state.*/
            Keccak previousBranchStateRoot = CreateCheckpoint();

            InitBranch(newBranchStateRoot);

            bool readOnly    = (options & ProcessingOptions.ReadOnlyChain) != 0;
            int  blocksCount = suggestedBlocks.Count;

            Block[] processedBlocks = new Block[blocksCount];
            try
            {
                for (int i = 0; i < blocksCount; i++)
                {
                    if (blocksCount > 64 && i % 8 == 0)
                    {
                        if (_logger.IsInfo)
                        {
                            _logger.Info($"Processing part of a long blocks branch {i}/{blocksCount}. Block: {suggestedBlocks[i]}");
                        }
                    }

                    _witnessCollector.Reset();
                    (Block processedBlock, TxReceipt[] receipts) = ProcessOne(suggestedBlocks[i], options, blockTracer);
                    processedBlocks[i] = processedBlock;

                    // be cautious here as AuRa depends on processing
                    PreCommitBlock(newBranchStateRoot, suggestedBlocks[i].Number);
                    if (!readOnly)
                    {
                        _witnessCollector.Persist(processedBlock.Hash !);
                        BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(processedBlock, receipts));
                    }

                    // CommitBranch in parts if we have long running branch
                    bool isFirstInBatch = i == 0;
                    bool isLastInBatch  = i == blocksCount - 1;
                    bool isNotAtTheEdge = !isFirstInBatch && !isLastInBatch;
                    bool isCommitPoint  = i % MaxUncommittedBlocks == 0 && isNotAtTheEdge;
                    if (isCommitPoint && readOnly == false)
                    {
                        if (_logger.IsInfo)
                        {
                            _logger.Info($"Commit part of a long blocks branch {i}/{blocksCount}");
                        }
                        CommitBranch();
                        previousBranchStateRoot = CreateCheckpoint();
                        Keccak?newStateRoot = suggestedBlocks[i].StateRoot;
                        InitBranch(newStateRoot, false);
                    }
                }

                if (readOnly)
                {
                    RestoreBranch(previousBranchStateRoot);
                }
                else
                {
                    // TODO: move to branch processor
                    CommitBranch();
                }

                return(processedBlocks);
            }
            catch (Exception ex) // try to restore for all cost
            {
                _logger.Trace($"Encountered exception {ex} while processing blocks.");
                RestoreBranch(previousBranchStateRoot);
                throw;
            }
        }
コード例 #12
0
        private Block ProcessOne(Block suggestedBlock, ProcessingOptions options, IBlockTracer blockTracer)
        {
            if (_syncConfig.ValidateTree)
            {
                if (_logger.IsWarn)
                {
                    _logger.Warn("Collecting trie stats:");
                }
                TrieStats stats = _stateProvider.CollectStats();
                if (stats.MissingNodes > 0)
                {
                    if (_logger.IsError)
                    {
                        _logger.Error(stats.ToString());
                    }
                }
                else
                {
                    if (_logger.IsWarn)
                    {
                        _logger.Warn(stats.ToString());
                    }
                }
            }

            if (suggestedBlock.IsGenesis)
            {
                return(suggestedBlock);
            }

            if (_specProvider.DaoBlockNumber.HasValue && _specProvider.DaoBlockNumber.Value == suggestedBlock.Header.Number)
            {
                if (_logger.IsInfo)
                {
                    _logger.Info("Applying DAO transition");
                }
                ApplyDaoTransition();
            }

            Block block    = PrepareBlockForProcessing(suggestedBlock);
            var   receipts = ProcessTransactions(block, options, blockTracer);

            SetReceiptsRootAndBloom(block, receipts);
            ApplyMinerRewards(block, blockTracer);

            _stateProvider.Commit(_specProvider.GetSpec(block.Number));

            block.Header.StateRoot = _stateProvider.StateRoot;
            block.Header.Hash      = BlockHeader.CalculateHash(block.Header);
            if ((options & ProcessingOptions.NoValidation) == 0 && !_blockValidator.ValidateProcessedBlock(block, receipts, suggestedBlock))
            {
                if (_logger.IsError)
                {
                    _logger.Error($"Processed block is not valid {suggestedBlock.ToString(Block.Format.FullHashAndNumber)}");
                }
                throw new InvalidBlockException(suggestedBlock.Hash);
            }

            if ((options & ProcessingOptions.StoreReceipts) != 0)
            {
                StoreTxReceipts(block, receipts);
            }

            if ((options & ProcessingOptions.StoreTraces) != 0)
            {
                StoreTraces(blockTracer as ParityLikeBlockTracer);
            }

            BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(block));
            return(block);
        }