private void SetBlockHash(HashHeightPair nextBlockHash)
{
this.persistedCoinviewTip = nextBlockHash;
this.rocksDb.Put(new byte[] { blockTable }.Concat(blockHashKey).ToArray(), nextBlockHash.ToBytes());
}
public async Task ReorgedBlocksAreDeletedFromRepositoryIfReorgDetectedAsync()
{
this.chain = CreateChain(1000);
this.repositoryTipHashAndHeight = new HashHeightPair(this.chain.Genesis.HashBlock, 0);
this.blockStoreQueue = new BlockStoreQueue(this.chain, this.chainState, new StoreSettings(),
this.nodeLifetime, this.blockRepositoryMock.Object, new LoggerFactory(), new Mock <INodeStats>().Object);
await this.blockStoreQueue.InitializeAsync().ConfigureAwait(false);
this.chainState.ConsensusTip = this.chain.Tip;
// Sending 500 blocks to the queue.
for (int i = 1; i < 500; i++)
{
Block block = this.network.Consensus.ConsensusFactory.CreateBlock();
block.GetSerializedSize();
this.blockStoreQueue.AddToPending(new ChainedHeaderBlock(block, this.chain.GetBlock(i)));
}
// Create alternative chain with fork point at 450.
ChainedHeader prevBlock = this.chain.GetBlock(450);
var alternativeBlocks = new List <ChainedHeader>();
for (int i = 0; i < 100; i++)
{
BlockHeader header = this.network.Consensus.ConsensusFactory.CreateBlockHeader();
header.Nonce = RandomUtils.GetUInt32();
header.HashPrevBlock = prevBlock.HashBlock;
header.Bits = Target.Difficulty1;
var chainedHeader = new ChainedHeader(header, header.GetHash(), prevBlock);
alternativeBlocks.Add(chainedHeader);
prevBlock = chainedHeader;
}
ChainedHeader savedHeader = this.chain.Tip;
this.chain.SetTip(alternativeBlocks.Last());
this.chainState.ConsensusTip = this.chain.Tip;
// Present alternative chain and trigger save.
foreach (ChainedHeader header in alternativeBlocks)
{
Block block = this.network.Consensus.ConsensusFactory.CreateBlock();
block.GetSerializedSize();
if (header == alternativeBlocks.Last())
{
this.chainState.IsAtBestChainTip = true;
}
this.blockStoreQueue.AddToPending(new ChainedHeaderBlock(block, header));
}
await WaitUntilQueueIsEmptyAsync().ConfigureAwait(false);
this.chainState.IsAtBestChainTip = false;
// Make sure only longest chain is saved.
Assert.Equal(this.chain.Tip.Height, this.repositoryTotalBlocksSaved);
// Present a new longer chain that will reorg the repository.
this.chain.SetTip(savedHeader);
this.chainState.ConsensusTip = this.chain.Tip;
for (int i = 451; i <= this.chain.Height; i++)
{
Block block = this.network.Consensus.ConsensusFactory.CreateBlock();
block.GetSerializedSize();
if (i == this.chain.Height)
{
this.chainState.IsAtBestChainTip = true;
}
this.blockStoreQueue.AddToPending(new ChainedHeaderBlock(block, this.chain.GetBlock(i)));
}
await WaitUntilQueueIsEmptyAsync().ConfigureAwait(false);
// Make sure chain is saved.
Assert.Equal(this.chain.Tip.Height + alternativeBlocks.Count, this.repositoryTotalBlocksSaved);
Assert.Equal(alternativeBlocks.Count, this.repositoryTotalBlocksDeleted);
// Dispose block store.
this.nodeLifetime.StopApplication();
this.blockStoreQueue.Dispose();
}
public void SaveChanges(IList <UnspentOutput> unspentOutputs, HashHeightPair HashHeightPair,
HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null)
{
throw new NotImplementedException();
}
/// <inheritdoc />
public void SaveChanges(IList <UnspentOutput> outputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null)
{
Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
Guard.NotNull(outputs, nameof(outputs));
lock (this.lockobj)
{
if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
{
this.logger.LogDebug("{0}:'{1}'", nameof(this.blockHash), this.blockHash);
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.blockHash = nextBlockHash;
long utxoSkipDisk = 0;
var rewindData = new RewindData(oldBlockHash);
Dictionary <OutPoint, int> indexItems = null;
if (this.rewindDataIndexCache != null)
{
indexItems = new Dictionary <OutPoint, int>();
}
foreach (UnspentOutput output in outputs)
{
if (!this.cachedUtxoItems.TryGetValue(output.OutPoint, out CacheItem cacheItem))
{
// Add outputs to cache, this will happen for two cases
// 1. if a chaced item was evicted
// 2. for new outputs that are added
if (output.CreatedFromBlock)
{
// if the output is indicate that it was added from a block
// There is no need to spend an extra call to disk.
this.logger.LogDebug("New Outpoint '{0}' created.", output.OutPoint);
cacheItem = new CacheItem()
{
ExistInInner = false,
IsDirty = false,
OutPoint = output.OutPoint,
Coins = null
};
}
else
{
// This can happen if the cashe item was evicted while
// the block was being processed, fetch the outut again from disk.
this.logger.LogDebug("Outpoint '{0}' is not found in cache, creating it.", output.OutPoint);
FetchCoinsResponse result = this.coindb.FetchCoins(new[] { output.OutPoint });
this.performanceCounter.AddMissCount(1);
UnspentOutput unspentOutput = result.UnspentOutputs.Single().Value;
cacheItem = new CacheItem()
{
ExistInInner = unspentOutput.Coins != null,
IsDirty = false,
OutPoint = unspentOutput.OutPoint,
Coins = unspentOutput.Coins
};
}
this.cachedUtxoItems.Add(cacheItem.OutPoint, cacheItem);
this.cacheSizeBytes += cacheItem.GetSize;
this.logger.LogDebug("CacheItem added to the cache during save '{0}'.", cacheItem.OutPoint);
}
// If output.Coins is null this means the utxo needs to be deleted
// otherwise this is a new utxo and we store it to cache.
if (output.Coins == null)
{
// DELETE COINS
// In cases of an output spent in the same block
// it wont exist in cash or in disk so its safe to remove it
if (cacheItem.Coins == null)
{
if (cacheItem.ExistInInner)
{
throw new InvalidOperationException(string.Format("Missmtch between coins in cache and in disk for output {0}", cacheItem.OutPoint));
}
}
else
{
// Handle rewind data
this.logger.LogDebug("Create restore outpoint '{0}' in OutputsToRestore rewind data.", cacheItem.OutPoint);
rewindData.OutputsToRestore.Add(new RewindDataOutput(cacheItem.OutPoint, cacheItem.Coins));
rewindData.TotalSize += cacheItem.GetSize;
if (this.rewindDataIndexCache != null && indexItems != null)
{
indexItems[cacheItem.OutPoint] = this.blockHash.Height;
}
}
// If a spent utxo never made it to disk then no need to keep it in memory.
if (!cacheItem.ExistInInner)
{
this.logger.LogDebug("Utxo '{0}' is not in disk, removing from cache.", cacheItem.OutPoint);
this.cachedUtxoItems.Remove(cacheItem.OutPoint);
this.cacheSizeBytes -= cacheItem.GetSize;
utxoSkipDisk++;
if (cacheItem.IsDirty)
{
this.dirtyCacheCount--;
}
}
else
{
// Now modify the cached items with the mutated data.
this.logger.LogDebug("Mark cache item '{0}' as spent .", cacheItem.OutPoint);
this.cacheSizeBytes -= cacheItem.GetScriptSize;
cacheItem.Coins = null;
// Delete output from cache but keep a the cache
// item reference so it will get deleted form disk
cacheItem.IsDirty = true;
this.dirtyCacheCount++;
}
}
else
{
// ADD COINS
if (cacheItem.Coins != null)
{
// Allow overrides.
// See https://github.com/bitcoin/bitcoin/blob/master/src/coins.cpp#L94
bool allowOverride = cacheItem.Coins.IsCoinbase && output.Coins != null;
if (!allowOverride)
{
throw new InvalidOperationException(string.Format("New coins override coins in cache or store, for output '{0}'", cacheItem.OutPoint));
}
this.logger.LogDebug("Coin override alllowed for utxo '{0}'.", cacheItem.OutPoint);
// Deduct the crurrent script size form the
// total cache size, it will be added again later.
this.cacheSizeBytes -= cacheItem.GetScriptSize;
// Clear this in order to calculate the cache sie
// this will get set later when overriden
cacheItem.Coins = null;
}
// Handle rewind data
// New trx so it needs to be deleted if a rewind happens.
this.logger.LogDebug("Adding output '{0}' to TransactionsToRemove rewind data.", cacheItem.OutPoint);
rewindData.OutputsToRemove.Add(cacheItem.OutPoint);
rewindData.TotalSize += cacheItem.GetSize;
// Put in the cache the new UTXOs.
this.logger.LogDebug("Mark cache item '{0}' as new .", cacheItem.OutPoint);
cacheItem.Coins = output.Coins;
this.cacheSizeBytes += cacheItem.GetScriptSize;
// Mark the cache item as dirty so it get persisted
// to disk and not evicted form cache
cacheItem.IsDirty = true;
this.dirtyCacheCount++;
}
}
this.performanceCounter.AddUtxoSkipDiskCount(utxoSkipDisk);
if (this.rewindDataIndexCache != null && indexItems.Any())
{
this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, indexItems);
}
// Add the most recent rewind data to the cache.
this.cachedRewindData.Add(this.blockHash.Height, rewindData);
this.rewindDataSizeBytes += rewindData.TotalSize;
}
}
public static MaturedBlockDepositsModel GetMaturedBlockDeposits(int depositCount = 0, HashHeightPair fixedHashHeight = null)
{
HashHeightPair hashHeightPair = fixedHashHeight ?? GetHashHeightPair();
IEnumerable <IDeposit> deposits = Enumerable.Range(0, depositCount).Select(_ => GetDeposit(hashHeightPair));
var maturedBlockDeposits = new MaturedBlockDepositsModel(
new MaturedBlockInfoModel()
{
BlockHash = hashHeightPair.Hash, BlockHeight = hashHeightPair.Height
},
deposits.ToList());
return(maturedBlockDeposits);
}
/// <summary>
/// Finds all changed records in the cache and persists them to the underlying coinview.
/// </summary>
/// <param name="force"><c>true</c> to enforce flush, <c>false</c> to flush only if <see cref="lastCacheFlushTime"/> is older than <see cref="CacheFlushTimeIntervalSeconds"/>.</param>
/// <remarks>
/// WARNING: This method can only be run from <see cref="ConsensusLoop.Execute(System.Threading.CancellationToken)"/> thread context
/// or when consensus loop is stopped. Otherwise, there is a risk of race condition when the consensus loop accepts new block.
/// </remarks>
public void Flush(bool force = true)
{
if (!force)
{
// Check if periodic flush is required.
// Ideally this will flush less frequent and always be behind
// blockstore which is currently set to 17 sec.
DateTime now = this.dateTimeProvider.GetUtcNow();
bool flushTimeLimit = (now - this.lastCacheFlushTime).TotalSeconds >= this.CacheFlushTimeIntervalSeconds;
// The size of the cache was reached and most likely TryEvictCacheLocked didn't work
// so the cahces is pulledted with flushable items, then we flush anyway.
long totalBytes = this.cacheSizeBytes + this.rewindDataSizeBytes;
bool flushSizeLimit = totalBytes > this.MaxCacheSizeBytes;
if (!flushTimeLimit && !flushSizeLimit)
{
return;
}
this.logger.LogDebug("Flushing, reasons flushTimeLimit={0} flushSizeLimit={1}.", flushTimeLimit, flushSizeLimit);
}
// Before flushing the coinview persist the stake store
// the stake store depends on the last block hash
// to be stored after the stake store is persisted.
if (this.stakeChainStore != null)
{
this.stakeChainStore.Flush(true);
}
// Before flushing the coinview persist the rewind data index store as well.
if (this.rewindDataIndexCache != null)
{
this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, null);
}
if (this.innerBlockHash == null)
{
this.innerBlockHash = this.coindb.GetTipHash();
}
lock (this.lockobj)
{
if (this.innerBlockHash == null)
{
this.logger.LogTrace("(-)[NULL_INNER_TIP]");
return;
}
var modify = new List <UnspentOutput>();
foreach (var cacheItem in this.cachedUtxoItems.Where(u => u.Value.IsDirty))
{
cacheItem.Value.IsDirty = false;
cacheItem.Value.ExistInInner = true;
modify.Add(new UnspentOutput(cacheItem.Key, cacheItem.Value.Coins));
}
this.logger.LogDebug("Flushing {0} items.", modify.Count);
this.coindb.SaveChanges(modify, this.innerBlockHash, this.blockHash, this.cachedRewindData.Select(c => c.Value).ToList());
// All the cached utxos are now on disk so we can clear the cached entry list.
this.cachedUtxoItems.Clear();
this.cacheSizeBytes = 0;
this.cachedRewindData.Clear();
this.rewindDataSizeBytes = 0;
this.dirtyCacheCount = 0;
this.innerBlockHash = this.blockHash;
}
this.lastCacheFlushTime = this.dateTimeProvider.GetUtcNow();
}
public void SaveChanges(IList <UnspentOutput> unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List <RewindData> rewindDataList = null)
{
int insertedEntities = 0;
using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction())
{
transaction.ValuesLazyLoadingIsOn = false;
transaction.SynchronizeTables("BlockHash", "Coins", "Rewind");
// Speed can degrade when keys are in random order and, especially, if these keys have high entropy.
// This settings helps with speed, see dBreeze documentations about details.
// We should double check if this settings help in our scenario, or sorting keys and operations is enough.
// Refers to issue #2483. https://github.com/stratisproject/StratisBitcoinFullNode/issues/2483
transaction.Technical_SetTable_OverwriteIsNotAllowed("Coins");
using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o)))
{
HashHeightPair current = this.GetTipHash(transaction);
if (current != oldBlockHash)
{
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.SetBlockHash(transaction, nextBlockHash);
// Here we'll add items to be inserted in a second pass.
List <UnspentOutput> toInsert = new List <UnspentOutput>();
foreach (var coin in unspentOutputs.OrderBy(utxo => utxo.OutPoint, new OutPointComparer()))
{
if (coin.Coins == null)
{
this.logger.LogDebug("Outputs of transaction ID '{0}' are prunable and will be removed from the database.", coin.OutPoint);
transaction.RemoveKey("Coins", coin.OutPoint.ToBytes());
}
else
{
// Add the item to another list that will be used in the second pass.
// This is for performance reasons: dBreeze is optimized to run the same kind of operations, sorted.
toInsert.Add(coin);
}
}
for (int i = 0; i < toInsert.Count; i++)
{
var coin = toInsert[i];
this.logger.LogDebug("Outputs of transaction ID '{0}' are NOT PRUNABLE and will be inserted into the database. {1}/{2}.", coin.OutPoint, i, toInsert.Count);
transaction.Insert("Coins", coin.OutPoint.ToBytes(), this.dataStoreSerializer.Serialize(coin.Coins));
}
if (rewindDataList != null)
{
foreach (RewindData rewindData in rewindDataList)
{
var nextRewindIndex = rewindData.PreviousBlockHash.Height + 1;
this.logger.LogDebug("Rewind state #{0} created.", nextRewindIndex);
transaction.Insert("Rewind", nextRewindIndex, this.dataStoreSerializer.Serialize(rewindData));
}
}
insertedEntities += unspentOutputs.Count;
transaction.Commit();
}
}
this.performanceCounter.AddInsertedEntities(insertedEntities);
}
private void SetBlockHash(DBreeze.Transactions.Transaction transaction, HashHeightPair nextBlockHash)
{
this.blockHash = nextBlockHash;
transaction.Insert <byte[], byte[]>("BlockHash", blockHashKey, nextBlockHash.ToBytes());
}
/// <summary>
/// Set's the hash and height tip of the new <see cref="ProvenBlockHeader"/>.
/// </summary>
/// <param name="transaction"> Open DBreeze transaction.</param>
/// <param name="newTip"> Hash height pair of the new block tip.</param>
private void SetTip(DBreeze.Transactions.Transaction transaction, HashHeightPair newTip)
{
Guard.NotNull(newTip, nameof(newTip));
transaction.Insert <byte[], HashHeightPair>(BlockHashHeightTable, blockHashHeightKey, newTip);
}
private void SaveTip(HashHeightPair tipToSave, DBreeze.Transactions.Transaction transaction)
{
transaction.Insert <byte[], byte[]>(TableName, TipKey.ToBytes(), this.dBreezeSerializer.Serialize(tipToSave));
}
