public Block[] Process(Keccak newBranchStateRoot, List <Block> suggestedBlocks, ProcessingOptions processingOptions, IBlockTracer blockTracer)
{
_logger.Info($"Processing {suggestedBlocks.Last().ToString(Block.Format.Short)}");
while (true)
{
bool notYet = false;
for (int i = 0; i < suggestedBlocks.Count; i++)
{
Keccak hash = suggestedBlocks[i].Hash;
if (!_allowed.Contains(hash))
{
if (_allowedToFail.Contains(hash))
{
_allowedToFail.Remove(hash);
BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(suggestedBlocks.Last()));
throw new InvalidBlockException(hash);
}
notYet = true;
break;
}
}
if (notYet)
{
Thread.Sleep(20);
}
else
{
BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(suggestedBlocks.Last()));
return(suggestedBlocks.ToArray());
}
}
}
private Block ProcessOne(Block suggestedBlock, ProcessingOptions options, IBlockTracer blockTracer)
{
Block block;
if (suggestedBlock.IsGenesis)
{
block = suggestedBlock;
}
else
{
if (_specProvider.DaoBlockNumber.HasValue && _specProvider.DaoBlockNumber.Value == suggestedBlock.Header.Number)
{
if (_logger.IsInfo)
{
_logger.Info("Applying DAO transition");
}
ApplyDaoTransition();
}
block = PrepareBlockForProcessing(suggestedBlock);
_additionalBlockProcessor?.PreProcess(block, options);
var receipts = ProcessTransactions(block, options, blockTracer);
SetReceiptsRoot(block, receipts);
ApplyMinerRewards(block, blockTracer);
_stateProvider.Commit(_specProvider.GetSpec(block.Number));
block.Header.StateRoot = _stateProvider.StateRoot;
block.Header.Hash = BlockHeader.CalculateHash(block.Header);
_additionalBlockProcessor?.PostProcess(block, receipts, options);
if ((options & ProcessingOptions.NoValidation) == 0 && !_blockValidator.ValidateProcessedBlock(block, receipts, suggestedBlock))
{
if (_logger.IsError)
{
_logger.Error($"Processed block is not valid {suggestedBlock.ToString(Block.Format.FullHashAndNumber)}");
}
// if (_logger.IsError) _logger.Error($"State: {_stateProvider.DumpState()}");
throw new InvalidBlockException(suggestedBlock.Hash);
}
if ((options & ProcessingOptions.StoreReceipts) != 0)
{
StoreTxReceipts(block, receipts);
}
if ((options & ProcessingOptions.StoreTraces) != 0)
{
StoreTraces(blockTracer as ParityLikeBlockTracer);
}
}
if ((options & ProcessingOptions.ReadOnlyChain) == 0)
{
BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(block));
}
return(block);
}
protected override void ProccessingData(object indexThread)
{
try
{
byte[] hederBytes;
int lenghtBlock;
int dataLenght;
byte[] data;
long position;
long sizefile;
while (true && !IsError)
{
BlockData block;
if (BlockReaded.TryTakeBlock(out block))
{
using (var memStream = new MemoryStream(block.Bytes))
{
using (var gzipStream = new GZipStream(memStream, CompressionMode.Decompress))
{
hederBytes = new byte[4] {
block.Bytes[4], block.Bytes[5], block.Bytes[6], block.Bytes[7]
};
lenghtBlock = BitConverter.ToInt32(hederBytes, 0);
dataLenght = BitConverter.ToInt32(block.Bytes, lenghtBlock - 4);
data = new byte[dataLenght];
gzipStream.Read(data, 0, dataLenght);
//отделяем от data первые 8 байт - 4 байта позиция записи, 4 байта размер файла
data = GetHelpersData(out position, out sizefile, data);
BlockProcessed.AddBlock(new BlockData(position, sizefile, data));
BlocksProcessedCount++;
}
}
}
else
{
if (BlockReaded.IsFinish && BlocksProcessedCount == BlocksCount)
{
BlockProcessed.Finish();
}
return;
}
}
}
catch (IOException e)
{
ErrorOutput(e, "Unable to complete decompression operation due to insufficient RAM. Please close other applications and try again. ");
}
catch (OutOfMemoryException e)
{
ErrorOutput(e, "Not enough RAM to complete file decompression. Please close other applications and try again. ");
}
catch (Exception e)
{
ErrorOutput(e);
}
finally
{
EventWaitHandleArray[(int)indexThread].Set();
}
}
// TODO: move to branch processor
public Block[] Process(Keccak newBranchStateRoot, List <Block> suggestedBlocks, ProcessingOptions options, IBlockTracer blockTracer)
{
if (suggestedBlocks.Count == 0)
{
return(Array.Empty <Block>());
}
/* We need to save the snapshot state root before reorganization in case the new branch has invalid blocks.
* In case of invalid blocks on the new branch we will discard the entire branch and come back to
* the previous head state.*/
Keccak previousBranchStateRoot = CreateCheckpoint();
InitBranch(newBranchStateRoot);
bool readOnly = (options & ProcessingOptions.ReadOnlyChain) != 0;
Block[] processedBlocks = new Block[suggestedBlocks.Count];
try
{
for (int i = 0; i < suggestedBlocks.Count; i++)
{
if (suggestedBlocks.Count > 64 && i % 8 == 0)
{
if (_logger.IsInfo)
{
_logger.Info($"Processing part of a long blocks branch {i}/{suggestedBlocks.Count}");
}
}
var(processedBlock, receipts) = ProcessOne(suggestedBlocks[i], options, blockTracer);
processedBlocks[i] = processedBlock;
// be cautious here as AuRa depends on processing
PreCommitBlock(newBranchStateRoot); // only needed if we plan to read state root?
if (!readOnly)
{
BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(processedBlock, receipts));
}
}
if (readOnly)
{
RestoreBranch(previousBranchStateRoot);
}
else
{
// TODO: move to branch processor
CommitBranch();
}
return(processedBlocks);
}
catch (Exception) // try to restore for all cost
{
RestoreBranch(previousBranchStateRoot);
throw;
}
}
protected override void WriteData()
{
try
{
int blocksWrite = 0;
BlockData block;
byte[] lenghtOfBlock;
while (true && !IsError)
{
using (var outputStream = new FileStream(OutputFile, FileMode.Append, FileAccess.Write))
{
if (BlockProcessed.TryTakeBlock(out block))
{
//получим размер сжатых данных для последующей декомпрессии
lenghtOfBlock = BitConverter.GetBytes(block.Bytes.Length);
/*запишем информацию о размере блока для последующей декомперссии и записи, вместо
* времени модификации файла в формате MTIME (согласно спецификации gzip) */
lenghtOfBlock.CopyTo(block.Bytes, 4);
outputStream.Write(block.Bytes, 0, block.Bytes.Length);
blocksWrite++;
ProgressInfo.Output(blocksWrite, BlocksCount);
}
else
{
if (blocksWrite == BlocksCount)
{
ProgressInfo.End();
}
else
{
throw new Exception("Can't write all blocks");
}
return;
}
}
}
}
catch (IOException e)
{
ErrorOutput(e, "Unable to complete write operation due to insufficient RAM. Please close other applications and try again. ");
}
catch (OutOfMemoryException e)
{
ErrorOutput(e, "Not enough RAM to complete file write. Please close other applications and try again. ");
}
catch (Exception e)
{
ErrorOutput(e);
}
finally
{
EventWaitHandleWrite.Set();
}
}
protected override void WriteData()
{
try
{
bool isSetLenght = false;
int blocksWrite = 0;
while (true && !IsError)
{
BlockData block;
if (BlockProcessed.TryTakeBlock(out block))
{
using (var outputStream = new FileStream(OutputFile, FileMode.Open, FileAccess.Write))
{
if (!isSetLenght)
{
outputStream.SetLength(block.SizeFile);
isSetLenght = true;
}
outputStream.Seek(block.Position, SeekOrigin.Begin);
outputStream.Write(block.Bytes, 0, block.Bytes.Length);
blocksWrite++;
ProgressInfo.Output(blocksWrite, BlocksCount);
}
}
else
{
if (blocksWrite >= BlocksCount)
{
ProgressInfo.End();
}
else
{
throw new Exception("Can't write all blocks");
}
return;
}
}
}
catch (IOException e)
{
ErrorOutput(e, "Unable to complete write operation due to insufficient RAM. Please close other applications and try again. ");
}
catch (OutOfMemoryException e)
{
ErrorOutput(e, "Not enough RAM to complete file write. Please close other applications and try again. ");
}
catch (Exception e)
{
ErrorOutput(e);
}
finally
{
EventWaitHandleWrite.Set();
}
}
public Block[] Process(Keccak newBranchStateRoot, List <Block> suggestedBlocks, ProcessingOptions options, IBlockTracer blockTracer)
{
if (suggestedBlocks.Count == 0)
{
return(Array.Empty <Block>());
}
/* We need to save the snapshot state root before reorganization in case the new branch has invalid blocks.
* In case of invalid blocks on the new branch we will discard the entire branch and come back to
* the previous head state.*/
Keccak previousBranchStateRoot = CreateCheckpoint();
InitBranch(newBranchStateRoot);
bool readOnly = (options & ProcessingOptions.ReadOnlyChain) != 0;
Block[] processedBlocks = new Block[suggestedBlocks.Count];
try
{
for (int i = 0; i < suggestedBlocks.Count; i++)
{
processedBlocks[i] = ProcessOne(suggestedBlocks[i], options, blockTracer);
// be cautious here as AuRa depends on processing
PreCommitBlock(newBranchStateRoot); // only needed if we plan to read state root?
if (!readOnly)
{
BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(processedBlocks[i]));
}
}
if (readOnly)
{
RestoreBranch(previousBranchStateRoot);
}
else
{
CommitBranch();
}
return(processedBlocks);
}
catch (InvalidBlockException)
{
RestoreBranch(previousBranchStateRoot);
throw;
}
}
/// <summary>
/// сжатие данных
/// </summary>
protected override void ProccessingData(object indexThread)
{
try
{
BlockData block;
while (true && !IsError)
{
if (BlockReaded.TryTakeBlock(out block))
{
using (var memStream = new MemoryStream())
{
using (var gzipStream = new GZipStream(memStream, CompressionMode.Compress))
{
gzipStream.Write(block.Bytes, 0, block.Bytes.Length);
}
BlockProcessed.AddBlock(new BlockData(memStream.ToArray()));
BlocksProcessedCount++;
}
}
else
{
if (BlockReaded.IsFinish && BlocksProcessedCount == BlocksCount)
{
BlockProcessed.Finish();
}
return;
}
}
}
catch (IOException e)
{
ErrorOutput(e, "Unable to complete compression operation due to insufficient RAM. Please close other applications and try again. ");
}
catch (OutOfMemoryException e)
{
ErrorOutput(e, "Not enough RAM to complete file compression. Please close other applications and try again. ");
}
catch (Exception e)
{
ErrorOutput(e);
}
finally
{
EventWaitHandleArray[(int)indexThread].Set();
}
}
public Block[] Process(Keccak newBranchStateRoot, List <Block> suggestedBlocks, ProcessingOptions processingOptions, IBlockTracer blockTracer)
{
if (blockTracer != NullBlockTracer.Instance)
{
// this is for block reruns on failure for diag tracing
throw new InvalidBlockException(suggestedBlocks[0]);
}
_logger.Info($"Processing {suggestedBlocks.Last().ToString(Block.Format.Short)}");
while (true)
{
bool notYet = false;
for (int i = 0; i < suggestedBlocks.Count; i++)
{
BlocksProcessing?.Invoke(this, new BlocksProcessingEventArgs(suggestedBlocks));
Block suggestedBlock = suggestedBlocks[i];
Keccak hash = suggestedBlock.Hash !;
if (!_allowed.Contains(hash))
{
if (_allowedToFail.Contains(hash))
{
_allowedToFail.Remove(hash);
BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(suggestedBlocks.Last(), Array.Empty <TxReceipt>()));
throw new InvalidBlockException(suggestedBlock);
}
notYet = true;
break;
}
}
if (notYet)
{
Thread.Sleep(20);
}
else
{
BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(suggestedBlocks.Last(), Array.Empty <TxReceipt>()));
return(suggestedBlocks.ToArray());
}
}
}
public Block[] Process(Keccak branchStateRoot, Block[] suggestedBlocks, ProcessingOptions options, IBlockTracer blockTracer)
{
if (suggestedBlocks.Length == 0)
{
return(Array.Empty <Block>());
}
int stateSnapshot = _stateDb.TakeSnapshot();
int codeSnapshot = _codeDb.TakeSnapshot();
if (stateSnapshot != -1 || codeSnapshot != -1)
{
if (_logger.IsError)
{
_logger.Error($"Uncommitted state ({stateSnapshot}, {codeSnapshot}) when processing from a branch root {branchStateRoot} starting with block {suggestedBlocks[0].ToString(Block.Format.Short)}");
}
}
Keccak snapshotStateRoot = _stateProvider.StateRoot;
if (branchStateRoot != null && _stateProvider.StateRoot != branchStateRoot)
{
/* discarding the other branch data - chain reorganization */
Metrics.Reorganizations++;
_storageProvider.Reset();
_stateProvider.Reset();
_stateProvider.StateRoot = branchStateRoot;
}
var readOnly = (options & ProcessingOptions.ReadOnlyChain) != 0;
var processedBlocks = new Block[suggestedBlocks.Length];
try
{
for (int i = 0; i < suggestedBlocks.Length; i++)
{
processedBlocks[i] = ProcessOne(suggestedBlocks[i], options, blockTracer);
if (_logger.IsTrace)
{
_logger.Trace($"Committing trees - state root {_stateProvider.StateRoot}");
}
_stateProvider.CommitTree();
_storageProvider.CommitTrees();
if (!readOnly)
{
BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(processedBlocks[i]));
}
}
if (readOnly)
{
_receiptsTracer.BeforeRestore(_stateProvider);
Restore(stateSnapshot, codeSnapshot, snapshotStateRoot);
}
else
{
_stateDb.Commit();
_codeDb.Commit();
}
return(processedBlocks);
}
catch (InvalidBlockException)
{
Restore(stateSnapshot, codeSnapshot, snapshotStateRoot);
throw;
}
}
// TODO: move to branch processor
public Block[] Process(Keccak newBranchStateRoot, List <Block> suggestedBlocks, ProcessingOptions options, IBlockTracer blockTracer)
{
if (suggestedBlocks.Count == 0)
{
return(Array.Empty <Block>());
}
BlocksProcessing?.Invoke(this, new BlocksProcessingEventArgs(suggestedBlocks));
/* We need to save the snapshot state root before reorganization in case the new branch has invalid blocks.
* In case of invalid blocks on the new branch we will discard the entire branch and come back to
* the previous head state.*/
Keccak previousBranchStateRoot = CreateCheckpoint();
InitBranch(newBranchStateRoot);
bool readOnly = (options & ProcessingOptions.ReadOnlyChain) != 0;
int blocksCount = suggestedBlocks.Count;
Block[] processedBlocks = new Block[blocksCount];
try
{
for (int i = 0; i < blocksCount; i++)
{
if (blocksCount > 64 && i % 8 == 0)
{
if (_logger.IsInfo)
{
_logger.Info($"Processing part of a long blocks branch {i}/{blocksCount}. Block: {suggestedBlocks[i]}");
}
}
_witnessCollector.Reset();
(Block processedBlock, TxReceipt[] receipts) = ProcessOne(suggestedBlocks[i], options, blockTracer);
processedBlocks[i] = processedBlock;
// be cautious here as AuRa depends on processing
PreCommitBlock(newBranchStateRoot, suggestedBlocks[i].Number);
if (!readOnly)
{
_witnessCollector.Persist(processedBlock.Hash !);
BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(processedBlock, receipts));
}
// CommitBranch in parts if we have long running branch
bool isFirstInBatch = i == 0;
bool isLastInBatch = i == blocksCount - 1;
bool isNotAtTheEdge = !isFirstInBatch && !isLastInBatch;
bool isCommitPoint = i % MaxUncommittedBlocks == 0 && isNotAtTheEdge;
if (isCommitPoint && readOnly == false)
{
if (_logger.IsInfo)
{
_logger.Info($"Commit part of a long blocks branch {i}/{blocksCount}");
}
CommitBranch();
previousBranchStateRoot = CreateCheckpoint();
Keccak?newStateRoot = suggestedBlocks[i].StateRoot;
InitBranch(newStateRoot, false);
}
}
if (readOnly)
{
RestoreBranch(previousBranchStateRoot);
}
else
{
// TODO: move to branch processor
CommitBranch();
}
return(processedBlocks);
}
catch (Exception ex) // try to restore for all cost
{
_logger.Trace($"Encountered exception {ex} while processing blocks.");
RestoreBranch(previousBranchStateRoot);
throw;
}
}
private Block ProcessOne(Block suggestedBlock, ProcessingOptions options, IBlockTracer blockTracer)
{
if (_syncConfig.ValidateTree)
{
if (_logger.IsWarn)
{
_logger.Warn("Collecting trie stats:");
}
TrieStats stats = _stateProvider.CollectStats();
if (stats.MissingNodes > 0)
{
if (_logger.IsError)
{
_logger.Error(stats.ToString());
}
}
else
{
if (_logger.IsWarn)
{
_logger.Warn(stats.ToString());
}
}
}
if (suggestedBlock.IsGenesis)
{
return(suggestedBlock);
}
if (_specProvider.DaoBlockNumber.HasValue && _specProvider.DaoBlockNumber.Value == suggestedBlock.Header.Number)
{
if (_logger.IsInfo)
{
_logger.Info("Applying DAO transition");
}
ApplyDaoTransition();
}
Block block = PrepareBlockForProcessing(suggestedBlock);
var receipts = ProcessTransactions(block, options, blockTracer);
SetReceiptsRootAndBloom(block, receipts);
ApplyMinerRewards(block, blockTracer);
_stateProvider.Commit(_specProvider.GetSpec(block.Number));
block.Header.StateRoot = _stateProvider.StateRoot;
block.Header.Hash = BlockHeader.CalculateHash(block.Header);
if ((options & ProcessingOptions.NoValidation) == 0 && !_blockValidator.ValidateProcessedBlock(block, receipts, suggestedBlock))
{
if (_logger.IsError)
{
_logger.Error($"Processed block is not valid {suggestedBlock.ToString(Block.Format.FullHashAndNumber)}");
}
throw new InvalidBlockException(suggestedBlock.Hash);
}
if ((options & ProcessingOptions.StoreReceipts) != 0)
{
StoreTxReceipts(block, receipts);
}
if ((options & ProcessingOptions.StoreTraces) != 0)
{
StoreTraces(blockTracer as ParityLikeBlockTracer);
}
BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(block));
return(block);
}