public void Has_ended_returns_true_when_ended()
{
MeasuredProgress measuredProgress = new MeasuredProgress();
measuredProgress.MarkEnd();
Assert.True(measuredProgress.HasEnded);
}
public void After_ending_does_not_update_total_or_current()
{
MeasuredProgress measuredProgress = new MeasuredProgress();
measuredProgress.Update(0L);
measuredProgress.SetMeasuringPoint();
Thread.Sleep(100);
measuredProgress.Update(1L);
measuredProgress.SetMeasuringPoint();
Thread.Sleep(100);
measuredProgress.Update(3L);
measuredProgress.MarkEnd();
Thread.Sleep(100);
measuredProgress.SetMeasuringPoint();
Thread.Sleep(100);
measuredProgress.SetMeasuringPoint();
Thread.Sleep(100);
measuredProgress.SetMeasuringPoint();
Assert.GreaterOrEqual(measuredProgress.TotalPerSecond, 14M);
Assert.AreEqual(0M, measuredProgress.CurrentPerSecond);
}
public void After_ending_does_not_update_total_or_current()
{
ManualTimestamper manualTimestamper = new ManualTimestamper();
MeasuredProgress measuredProgress = new MeasuredProgress(manualTimestamper);
measuredProgress.Update(0L);
measuredProgress.SetMeasuringPoint();
manualTimestamper.Add(TimeSpan.FromMilliseconds(100));
measuredProgress.Update(1L);
measuredProgress.SetMeasuringPoint();
manualTimestamper.Add(TimeSpan.FromMilliseconds(100));
measuredProgress.Update(3L);
measuredProgress.MarkEnd();
manualTimestamper.Add(TimeSpan.FromMilliseconds(100));
measuredProgress.SetMeasuringPoint();
manualTimestamper.Add(TimeSpan.FromMilliseconds(100));
measuredProgress.SetMeasuringPoint();
manualTimestamper.Add(TimeSpan.FromMilliseconds(100));
measuredProgress.SetMeasuringPoint();
Assert.GreaterOrEqual(measuredProgress.TotalPerSecond, 6M);
Assert.LessOrEqual(measuredProgress.TotalPerSecond, 15M);
Assert.AreEqual(0M, measuredProgress.CurrentPerSecond);
}
private void RunBloomMigration(CancellationToken token)
{
BlockHeader GetMissingBlockHeader(long i)
{
if (_logger.IsWarn)
{
_logger.Warn(GetLogMessage("warning", $"Header for block {i} not found. Logs will not be searchable for this block."));
}
return(EmptyHeader);
}
if (_api.BloomStorage == null)
{
throw new StepDependencyException(nameof(_api.BloomStorage));
}
if (_api.BlockTree == null)
{
throw new StepDependencyException(nameof(_api.BlockTree));
}
if (_api.ChainLevelInfoRepository == null)
{
throw new StepDependencyException(nameof(_api.ChainLevelInfoRepository));
}
IBlockTree blockTree = _api.BlockTree;
IBloomStorage storage = _api.BloomStorage;
long to = MinBlockNumber;
long synced = storage.MigratedBlockNumber + 1;
long from = synced;
_migrateCount = to + 1;
_averages = _api.BloomStorage.Averages.ToArray();
IChainLevelInfoRepository?chainLevelInfoRepository = _api.ChainLevelInfoRepository;
_progress.Update(synced);
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("started"));
}
using (var timer = new Timer(1000)
{
Enabled = true
})
{
timer.Elapsed += (ElapsedEventHandler)((o, e) =>
{
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("in progress"));
}
});
try
{
storage.Migrate(GetHeadersForMigration());
}
finally
{
_progress.MarkEnd();
_stopwatch?.Stop();
}
IEnumerable <BlockHeader> GetHeadersForMigration()
{
bool TryGetMainChainBlockHashFromLevel(long number, out Keccak?blockHash)
{
using var batch = chainLevelInfoRepository.StartBatch();
var level = chainLevelInfoRepository.LoadLevel(number);
if (level != null)
{
if (!level.HasBlockOnMainChain)
{
if (level.BlockInfos.Length > 0)
{
level.HasBlockOnMainChain = true;
chainLevelInfoRepository.PersistLevel(number, level, batch);
}
}
blockHash = level.MainChainBlock?.BlockHash;
return(blockHash != null);
}
else
{
blockHash = null;
return(false);
}
}
for (long i = from; i <= to; i++)
{
if (token.IsCancellationRequested)
{
timer.Stop();
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("cancelled"));
}
yield break;
}
if (TryGetMainChainBlockHashFromLevel(i, out Keccak? blockHash))
{
var header = blockTree.FindHeader(blockHash, BlockTreeLookupOptions.None);
yield return(header ?? GetMissingBlockHeader(i));
}
else
{
yield return(GetMissingBlockHeader(i));
}
_progress.Update(++synced);
}
}
}
if (!token.IsCancellationRequested)
{
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("finished"));
}
}
}
private void RunMigration(CancellationToken token)
{
Block GetMissingBlock(long i, Keccak?blockHash)
{
if (_logger.IsWarn)
{
_logger.Warn(GetLogMessage("warning", $"Block {i} not found. Logs will not be searchable for this block."));
}
EmptyBlock.Header.Number = i;
EmptyBlock.Header.Hash = blockHash;
return(EmptyBlock);
}
long synced = 0;
IDb receiptsDb = _dbProvider.ReceiptsDb;
_progress.Update(synced);
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("started"));
}
using (var timer = new Timer(1000)
{
Enabled = true
})
{
timer.Elapsed += (ElapsedEventHandler)((o, e) =>
{
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("in progress"));
}
});
try
{
foreach (var block in GetBlockBodiesForMigration())
{
var receipts = _receiptStorage.Get(block);
var notNullReceipts = receipts.Length == 0 ? receipts : receipts.Where(r => r != null).ToArray();
if (receipts.Length == 0 || notNullReceipts.Length != 0) // if notNullReceipts.Length is 0 and receipts are not 0 - we are missing all receipts, they are not processed yet.
{
_receiptStorage.Insert(block, notNullReceipts);
_receiptStorage.MigratedBlockNumber = block.Number;
for (int i = 0; i < notNullReceipts.Length; i++)
{
receiptsDb.Delete(notNullReceipts[i].TxHash);
}
if (notNullReceipts.Length != receipts.Length)
{
if (_logger.IsWarn)
{
_logger.Warn(GetLogMessage("warning", $"Block {block.ToString(Block.Format.FullHashAndNumber)} is missing {receipts.Length - notNullReceipts.Length} receipts!"));
}
}
}
}
}
finally
{
_progress.MarkEnd();
_stopwatch?.Stop();
}
IEnumerable <Block> GetBlockBodiesForMigration()
{
bool TryGetMainChainBlockHashFromLevel(long number, out Keccak?blockHash)
{
using var batch = _chainLevelInfoRepository.StartBatch();
var level = _chainLevelInfoRepository.LoadLevel(number);
if (level != null)
{
if (!level.HasBlockOnMainChain)
{
if (level.BlockInfos.Length > 0)
{
level.HasBlockOnMainChain = true;
_chainLevelInfoRepository.PersistLevel(number, level, batch);
}
}
blockHash = level.MainChainBlock?.BlockHash;
return(blockHash != null);
}
else
{
blockHash = null;
return(false);
}
}
for (long i = _toBlock - 1; i > 0; i--)
{
if (token.IsCancellationRequested)
{
timer.Stop();
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("cancelled"));
}
yield break;
}
if (TryGetMainChainBlockHashFromLevel(i, out var blockHash))
{
var header = _blockTree.FindBlock(blockHash, BlockTreeLookupOptions.None);
yield return(header ?? GetMissingBlock(i, blockHash));
}
_progress.Update(++synced);
}
}
}
if (!token.IsCancellationRequested)
{
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("finished"));
}
}
}
private void RunBloomMigration(CancellationToken token)
{
if (_context.BloomStorage == null)
{
throw new StepDependencyException(nameof(_context.BloomStorage));
}
if (_context.BlockTree == null)
{
throw new StepDependencyException(nameof(_context.BlockTree));
}
IBlockTree blockTree = _context.BlockTree;
IBloomStorage storage = _context.BloomStorage;
long to = MinBlockNumber;
long synced = storage.MigratedBlockNumber + 1;
long from = synced;
_migrateCount = to + 1;
_averages = _context.BloomStorage.Averages.ToArray();
_progress.Update(synced);
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("started"));
}
using (var timer = new Timer(1000)
{
Enabled = true
})
{
timer.Elapsed += (ElapsedEventHandler)((o, e) =>
{
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("in progress"));
}
});
try
{
storage.Migrate(GetHeadersForMigration());
}
finally
{
_progress.MarkEnd();
_stopwatch?.Stop();
}
IEnumerable <BlockHeader> GetHeadersForMigration()
{
for (long i = from; i <= to; i++)
{
if (token.IsCancellationRequested)
{
timer.Stop();
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("cancelled"));
}
yield break;
}
var header = blockTree.FindHeader(i);
yield return(header ?? new BlockHeader(Keccak.Zero, Keccak.Zero, Address.Zero, UInt256.Zero, 0L, 0L, UInt256.Zero, Bytes.Empty));
synced++;
_progress.Update(synced);
}
}
}
if (!token.IsCancellationRequested)
{
if (_logger.IsInfo)
{
_logger.Info(GetLogMessage("finished"));
}
}
}