public async Task TestRewindAsync()
{
uint256 tip = await this.cachedCoinView.GetTipHashAsync();
Assert.Equal(this.concurrentChain.Genesis.HashBlock, tip);
int currentHeight = 0;
// Create a lot of new coins.
List <UnspentOutputs> outputsList = this.CreateOutputsList(currentHeight + 1, 100);
await this.SaveChangesAsync(outputsList, new List <TxOut[]>(), currentHeight + 1);
currentHeight++;
await this.cachedCoinView.FlushAsync(true);
uint256 tipAfterOriginalCoinsCreation = await this.cachedCoinView.GetTipHashAsync();
// Collection that will be used as a coinview that we will update in parallel. Needed to verify that actual coinview is ok.
List <OutPoint> outPoints = this.ConvertToListOfOutputPoints(outputsList);
// Copy of current state to later rewind and verify against it.
List <OutPoint> copyOfOriginalOutPoints = new List <OutPoint>(outPoints);
List <OutPoint> copyAfterHalfOfAdditions = new List <OutPoint>();
uint256 coinviewTipAfterHalf = null;
int addChangesTimes = 500;
// Spend some coins in the next N saves.
for (int i = 0; i < addChangesTimes; ++i)
{
uint256 txId = outPoints[this.random.Next(0, outPoints.Count)].Hash;
List <OutPoint> txPoints = outPoints.Where(x => x.Hash == txId).ToList();
this.Shuffle(txPoints);
List <OutPoint> txPointsToSpend = txPoints.Take(txPoints.Count / 2).ToList();
// First spend in cached coinview
FetchCoinsResponse response = await this.cachedCoinView.FetchCoinsAsync(new[] { txId });
Assert.Single(response.UnspentOutputs);
UnspentOutputs coins = response.UnspentOutputs[0];
UnspentOutputs unchangedClone = coins.Clone();
foreach (OutPoint outPointToSpend in txPointsToSpend)
{
coins.Spend(outPointToSpend.N);
}
// Spend from outPoints.
outPoints.RemoveAll(x => txPointsToSpend.Contains(x));
// Save coinview
await this.SaveChangesAsync(new List <UnspentOutputs>() { coins }, new List <TxOut[]>() { unchangedClone.Outputs }, currentHeight + 1);
currentHeight++;
if (i == addChangesTimes / 2)
{
copyAfterHalfOfAdditions = new List <OutPoint>(outPoints);
coinviewTipAfterHalf = await this.cachedCoinView.GetTipHashAsync();
}
}
await this.ValidateCoinviewIntegrityAsync(outPoints);
for (int i = 0; i < addChangesTimes; i++)
{
await this.cachedCoinView.RewindAsync();
uint256 currentTip = await this.cachedCoinView.GetTipHashAsync();
if (currentTip == coinviewTipAfterHalf)
{
await this.ValidateCoinviewIntegrityAsync(copyAfterHalfOfAdditions);
}
}
Assert.Equal(tipAfterOriginalCoinsCreation, await this.cachedCoinView.GetTipHashAsync());
await this.ValidateCoinviewIntegrityAsync(copyOfOriginalOutPoints);
}
/// <inheritdoc />
public void SaveChanges(IList <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash, int height, List <RewindData> rewindDataList = null)
{
Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
Guard.NotNull(unspentOutputs, nameof(unspentOutputs));
lock (this.lockobj)
{
if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
{
this.logger.LogTrace("{0}:'{1}'", nameof(this.blockHash), this.blockHash);
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.blockHeight = height;
this.blockHash = nextBlockHash;
var rewindData = new RewindData(oldBlockHash);
var indexItems = new Dictionary <OutPoint, int>();
foreach (UnspentOutputs unspent in unspentOutputs)
{
if (!this.cachedUtxoItems.TryGetValue(unspent.TransactionId, out CacheItem cacheItem))
{
// This can happen very rarely in the case where we fetch items from
// disk but immediately call the Evict method which then removes the cached item(s).
this.logger.LogTrace("Outputs of transaction ID '{0}' are not found in cache, creating them.", unspent.TransactionId);
FetchCoinsResponse result = this.inner.FetchCoins(new[] { unspent.TransactionId });
UnspentOutputs unspentOutput = result.UnspentOutputs[0];
cacheItem = new CacheItem();
cacheItem.ExistInInner = unspentOutput != null;
cacheItem.IsDirty = false;
cacheItem.UnspentOutputs = unspentOutput?.Clone();
this.cachedUtxoItems.TryAdd(unspent.TransactionId, cacheItem);
this.logger.LogTrace("CacheItem added to the cache during save '{0}'.", cacheItem.UnspentOutputs);
}
// If cacheItem.UnspentOutputs is null this means the trx was not stored in the disk,
// that means the trx (and UTXO) is new and all the UTXOs need to be stored in cache
// otherwise we store to cache only the UTXO that have been spent.
if (cacheItem.UnspentOutputs != null)
{
// To handle rewind we'll need to restore the original outputs,
// so we clone it and save it in rewind data.
UnspentOutputs clone = unspent.Clone();
// We take the original items that are in cache and put them in rewind data.
clone.Outputs = cacheItem.UnspentOutputs.Outputs.ToArray();
this.logger.LogTrace("Modifying transaction '{0}' in OutputsToRestore rewind data.", unspent.TransactionId);
rewindData.OutputsToRestore.Add(clone);
this.logger.LogTrace("Cache item before spend {0}:'{1}'.", nameof(cacheItem.UnspentOutputs), cacheItem.UnspentOutputs);
// Now modify the cached items with the mutated data.
cacheItem.UnspentOutputs.Spend(unspent);
this.logger.LogTrace("Cache item after spend {0}:'{1}'.", nameof(cacheItem.UnspentOutputs), cacheItem.UnspentOutputs);
}
else
{
// New trx so it needs to be deleted if a rewind happens.
this.logger.LogTrace("Adding transaction '{0}' to TransactionsToRemove rewind data.", unspent.TransactionId);
rewindData.TransactionsToRemove.Add(unspent.TransactionId);
// Put in the cache the new UTXOs.
this.logger.LogTrace("Setting {0} to {1}: '{2}'.", nameof(cacheItem.UnspentOutputs), nameof(unspent), unspent);
cacheItem.UnspentOutputs = unspent;
}
cacheItem.IsDirty = true;
if (this.rewindDataIndexCache != null)
{
for (int i = 0; i < unspent.Outputs.Length; i++)
{
var key = new OutPoint(unspent.TransactionId, i);
indexItems[key] = this.blockHeight;
}
}
// Inner does not need to know pruned unspent that it never saw.
if (cacheItem.UnspentOutputs.IsPrunable && !cacheItem.ExistInInner)
{
this.logger.LogTrace("Outputs of transaction ID '{0}' are prunable and not in underlaying coinview, removing from cache.", unspent.TransactionId);
this.cachedUtxoItems.Remove(unspent.TransactionId);
}
}
if (this.rewindDataIndexCache != null && indexItems.Any())
{
this.rewindDataIndexCache.Save(indexItems);
this.rewindDataIndexCache.Flush(this.blockHeight);
}
this.cachedRewindDataIndex.Add(this.blockHeight, rewindData);
}
}
/// <inheritdoc />
public async Task SaveChangesAsync(IEnumerable <UnspentOutputs> unspentOutputs, IEnumerable <TxOut[]> originalOutputs, uint256 oldBlockHash, uint256 nextBlockHash, List <RewindData> rewindDataList = null)
{
Guard.NotNull(oldBlockHash, nameof(oldBlockHash));
Guard.NotNull(nextBlockHash, nameof(nextBlockHash));
Guard.NotNull(unspentOutputs, nameof(unspentOutputs));
this.logger.LogTrace("({0}.Count():{1},{2}.Count():{3},{4}:'{5}',{6}:'{7}')", nameof(unspentOutputs), unspentOutputs.Count(), nameof(originalOutputs), originalOutputs?.Count(), nameof(oldBlockHash), oldBlockHash, nameof(nextBlockHash), nextBlockHash);
using (await this.lockobj.LockAsync().ConfigureAwait(false))
{
if ((this.blockHash != null) && (oldBlockHash != this.blockHash))
{
this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]");
throw new InvalidOperationException("Invalid oldBlockHash");
}
this.blockHash = nextBlockHash;
var rewindData = new RewindData(oldBlockHash);
foreach (UnspentOutputs unspent in unspentOutputs)
{
if (!this.cachedUtxoItems.TryGetValue(unspent.TransactionId, out CacheItem cacheItem))
{
// This can happen very rarely in the case where we fetch items from
// disk but immediately call the Evict method which then removes the cached item(s).
this.logger.LogTrace("Outputs of transaction ID '{0}' are not found in cache, creating them.", unspent.TransactionId);
FetchCoinsResponse result = await this.inner.FetchCoinsAsync(new[] { unspent.TransactionId }).ConfigureAwait(false);
UnspentOutputs unspentOutput = result.UnspentOutputs[0];
cacheItem = new CacheItem();
cacheItem.ExistInInner = unspentOutput != null;
cacheItem.IsDirty = false;
cacheItem.UnspentOutputs = unspentOutput?.Clone();
this.cachedUtxoItems.TryAdd(unspent.TransactionId, cacheItem);
}
else
{
this.logger.LogTrace("Outputs of transaction ID '{0}' are in cache already, updating them.", unspent.TransactionId);
}
// If cacheItem.UnspentOutputs is null this means the trx was not stored in the disk,
// that means the trx (and UTXO) is new and all the UTXOs need to be stored in cache
// otherwise we store to cache only the UTXO that have been spent.
if (cacheItem.UnspentOutputs != null)
{
// To handle rewind we'll need to restore the original outputs,
// so we clone it and save it in rewind data.
UnspentOutputs clone = unspent.Clone();
// We take the original items that are in cache and put them in rewind data.
clone.Outputs = cacheItem.UnspentOutputs.Outputs.ToArray();
rewindData.OutputsToRestore.Add(clone);
// Now modify the cached items with the mutated data.
cacheItem.UnspentOutputs.Spend(unspent);
}
else
{
// New trx so it needs to be deleted if a rewind happens.
rewindData.TransactionsToRemove.Add(unspent.TransactionId);
// Put in the cache the new UTXOs.
cacheItem.UnspentOutputs = unspent;
}
cacheItem.IsDirty = true;
// Inner does not need to know pruned unspent that it never saw.
if (cacheItem.UnspentOutputs.IsPrunable && !cacheItem.ExistInInner)
{
this.logger.LogTrace("Outputs of transaction ID '{0}' are prunable and not in underlaying coinview, removing from cache.", unspent.TransactionId);
this.cachedUtxoItems.Remove(unspent.TransactionId);
}
}
this.cachedRewindDataList.Add(rewindData);
}
this.logger.LogTrace("(-)");
}
/// <inheritdoc />
public FetchCoinsResponse FetchCoins(uint256[] txIds, CancellationToken cancellationToken = default(CancellationToken))
{
Guard.NotNull(txIds, nameof(txIds));
FetchCoinsResponse result = null;
var outputs = new UnspentOutputs[txIds.Length];
var miss = new List <int>();
var missedTxIds = new List <uint256>();
lock (this.lockobj)
{
for (int i = 0; i < txIds.Length; i++)
{
CacheItem cache;
if (!this.cachedUtxoItems.TryGetValue(txIds[i], out cache))
{
this.logger.LogTrace("Transaction '{0}' not found in cache.", txIds[i]);
miss.Add(i);
missedTxIds.Add(txIds[i]);
}
else
{
this.logger.LogTrace("Transaction '{0}' found in cache, UTXOs:'{1}'.", txIds[i], cache.UnspentOutputs);
outputs[i] = cache.UnspentOutputs == null ? null :
cache.UnspentOutputs.IsPrunable ? null :
cache.UnspentOutputs.Clone();
}
}
this.performanceCounter.AddMissCount(miss.Count);
this.performanceCounter.AddHitCount(txIds.Length - miss.Count);
FetchCoinsResponse fetchedCoins = null;
if (missedTxIds.Count > 0 || this.blockHash == null)
{
this.logger.LogTrace("{0} cache missed transaction needs to be loaded from underlying CoinView.", missedTxIds.Count);
fetchedCoins = this.Inner.FetchCoins(missedTxIds.ToArray(), cancellationToken);
}
if (this.blockHash == null)
{
uint256 innerblockHash = fetchedCoins.BlockHash;
Debug.Assert(this.cachedUtxoItems.Count == 0);
this.innerBlockHash = innerblockHash;
this.blockHash = this.innerBlockHash;
}
for (int i = 0; i < miss.Count; i++)
{
int index = miss[i];
UnspentOutputs unspent = fetchedCoins.UnspentOutputs[i];
outputs[index] = unspent;
var cache = new CacheItem();
cache.ExistInInner = unspent != null;
cache.IsDirty = false;
cache.UnspentOutputs = unspent?.Clone();
this.logger.LogTrace("CacheItem added to the cache, Transaction Id '{0}', UTXO:'{1}'.", txIds[index], cache.UnspentOutputs);
this.cachedUtxoItems.TryAdd(txIds[index], cache);
}
result = new FetchCoinsResponse(outputs, this.blockHash);
int cacheEntryCount = this.cacheEntryCount;
if (cacheEntryCount > this.MaxItems)
{
this.logger.LogTrace("Cache is full now with {0} entries, evicting.", cacheEntryCount);
this.EvictLocked();
}
}
return(result);
}
/// <inheritdoc />
public async Task <FetchCoinsResponse> FetchCoinsAsync(uint256[] txIds, CancellationToken cancellationToken = default(CancellationToken))
{
Guard.NotNull(txIds, nameof(txIds));
this.logger.LogTrace("({0}.{1}:{2})", nameof(txIds), nameof(txIds.Length), txIds.Length);
FetchCoinsResponse result = null;
var outputs = new UnspentOutputs[txIds.Length];
var miss = new List <int>();
var missedTxIds = new List <uint256>();
using (await this.lockobj.LockAsync(cancellationToken).ConfigureAwait(false))
{
for (int i = 0; i < txIds.Length; i++)
{
CacheItem cache;
if (!this.cachedUtxoItems.TryGetValue(txIds[i], out cache))
{
this.logger.LogTrace("Cache missed for transaction ID '{0}'.", txIds[i]);
miss.Add(i);
missedTxIds.Add(txIds[i]);
}
else
{
this.logger.LogTrace("Cache hit for transaction ID '{0}'.", txIds[i]);
outputs[i] = cache.UnspentOutputs == null ? null :
cache.UnspentOutputs.IsPrunable ? null :
cache.UnspentOutputs.Clone();
}
}
this.PerformanceCounter.AddMissCount(miss.Count);
this.PerformanceCounter.AddHitCount(txIds.Length - miss.Count);
}
this.logger.LogTrace("{0} cache missed transaction needs to be loaded from underlying CoinView.", missedTxIds.Count);
FetchCoinsResponse fetchedCoins = await this.Inner.FetchCoinsAsync(missedTxIds.ToArray(), cancellationToken).ConfigureAwait(false);
using (await this.lockobj.LockAsync(cancellationToken).ConfigureAwait(false))
{
uint256 innerblockHash = fetchedCoins.BlockHash;
if (this.blockHash == null)
{
Debug.Assert(this.cachedUtxoItems.Count == 0);
this.innerBlockHash = innerblockHash;
this.blockHash = this.innerBlockHash;
}
for (int i = 0; i < miss.Count; i++)
{
int index = miss[i];
UnspentOutputs unspent = fetchedCoins.UnspentOutputs[i];
outputs[index] = unspent;
var cache = new CacheItem();
cache.ExistInInner = unspent != null;
cache.IsDirty = false;
cache.UnspentOutputs = unspent?.Clone();
this.cachedUtxoItems.TryAdd(txIds[index], cache);
}
result = new FetchCoinsResponse(outputs, this.blockHash);
}
int cacheEntryCount = this.CacheEntryCount;
if (cacheEntryCount > this.MaxItems)
{
this.logger.LogTrace("Cache is full now with {0} entries, evicting.", cacheEntryCount);
await this.EvictAsync().ConfigureAwait(false);
}
this.logger.LogTrace("(-):*.{0}='{1}',*.{2}.{3}={4}", nameof(result.BlockHash), result.BlockHash, nameof(result.UnspentOutputs), nameof(result.UnspentOutputs.Length), result.UnspentOutputs.Length);
return(result);
}
